Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(report): IgnoredJSONKyes to clear values in result json (#1071)

Browse Source 
* feat(report): IgnoredJSONKyes to clear values in result json

* fix(report): marshal indent in JSON everytime
This commit is contained in:
Kota Kanbe
2020-11-05 20:13:09 +09:00
committed by GitHub
parent 2fc3462d35
commit 93059b74c3
6 changed files with 211 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
commands/report.go
View File

@@ -235,85 +235,9 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
return subcommands.ExitFailure
}
// report
reports := []report.ResultWriter{
report.StdoutWriter{},
}
if c.Conf.ToSlack {
reports = append(reports, report.SlackWriter{})
}
if c.Conf.ToStride {
reports = append(reports, report.StrideWriter{})
}
if c.Conf.ToHipChat {
reports = append(reports, report.HipChatWriter{})
}
if c.Conf.ToChatWork {
reports = append(reports, report.ChatWorkWriter{})
}
if c.Conf.ToTelegram {
reports = append(reports, report.TelegramWriter{})
}
if c.Conf.ToEmail {
reports = append(reports, report.EMailWriter{})
}
if c.Conf.ToSyslog {
reports = append(reports, report.SyslogWriter{})
}
if c.Conf.ToHTTP {
reports = append(reports, report.HTTPRequestWriter{})
}
if c.Conf.ToLocalFile {
reports = append(reports, report.LocalFileWriter{
CurrentDir: dir,
})
}
if c.Conf.ToS3 {
if err := report.CheckIfBucketExists(); err != nil {
util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %+v",
c.Conf.AWS.S3Bucket, err)
return subcommands.ExitUsageError
}
reports = append(reports, report.S3Writer{})
}
if c.Conf.ToAzureBlob {
if len(c.Conf.Azure.AccountName) == 0 {
c.Conf.Azure.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
if len(c.Conf.Azure.AccountKey) == 0 {
c.Conf.Azure.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
}
if len(c.Conf.Azure.ContainerName) == 0 {
util.Log.Error("Azure storage container name is required with -azure-container option")
return subcommands.ExitUsageError
}
if err := report.CheckIfAzureContainerExists(); err != nil {
util.Log.Errorf("Check if there is a container beforehand: %s, err: %+v",
c.Conf.Azure.ContainerName, err)
return subcommands.ExitUsageError
}
reports = append(reports, report.AzureBlobWriter{})
}
if c.Conf.ToSaas {
if !c.Conf.UUID {
util.Log.Errorf("If you use the -to-saas option, you need to enable the uuid option")
return subcommands.ExitUsageError
}
reports = append(reports, report.SaasWriter{})
util.Log.Info("Validating config...")
if !c.Conf.ValidateOnReport() {
return subcommands.ExitUsageError
}
if !(c.Conf.FormatJSON || c.Conf.FormatOneLineText ||
@@ -321,11 +245,6 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
c.Conf.FormatList = true
}
util.Log.Info("Validating config...")
if !c.Conf.ValidateOnReport() {
return subcommands.ExitUsageError
}
var loaded models.ScanResults
if loaded, err = report.LoadScanResults(dir); err != nil {
util.Log.Error(err)
@@ -437,6 +356,87 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
}
}
// report
reports := []report.ResultWriter{
report.StdoutWriter{},
}
if c.Conf.ToSlack {
reports = append(reports, report.SlackWriter{})
}
if c.Conf.ToStride {
reports = append(reports, report.StrideWriter{})
}
if c.Conf.ToHipChat {
reports = append(reports, report.HipChatWriter{})
}
if c.Conf.ToChatWork {
reports = append(reports, report.ChatWorkWriter{})
}
if c.Conf.ToTelegram {
reports = append(reports, report.TelegramWriter{})
}
if c.Conf.ToEmail {
reports = append(reports, report.EMailWriter{})
}
if c.Conf.ToSyslog {
reports = append(reports, report.SyslogWriter{})
}
if c.Conf.ToHTTP {
reports = append(reports, report.HTTPRequestWriter{})
}
if c.Conf.ToLocalFile {
reports = append(reports, report.LocalFileWriter{
CurrentDir: dir,
})
}
if c.Conf.ToS3 {
if err := report.CheckIfBucketExists(); err != nil {
util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %+v",
c.Conf.AWS.S3Bucket, err)
return subcommands.ExitUsageError
}
reports = append(reports, report.S3Writer{})
}
if c.Conf.ToAzureBlob {
if len(c.Conf.Azure.AccountName) == 0 {
c.Conf.Azure.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
if len(c.Conf.Azure.AccountKey) == 0 {
c.Conf.Azure.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
}
if len(c.Conf.Azure.ContainerName) == 0 {
util.Log.Error("Azure storage container name is required with -azure-container option")
return subcommands.ExitUsageError
}
if err := report.CheckIfAzureContainerExists(); err != nil {
util.Log.Errorf("Check if there is a container beforehand: %s, err: %+v",
c.Conf.Azure.ContainerName, err)
return subcommands.ExitUsageError
}
reports = append(reports, report.AzureBlobWriter{})
}
if c.Conf.ToSaas {
if !c.Conf.UUID {
util.Log.Errorf("If you use the -to-saas option, you need to enable the uuid option")
return subcommands.ExitUsageError
}
reports = append(reports, report.SaasWriter{})
}
for _, w := range reports {
if err := w.Write(res...); err != nil {
util.Log.Errorf("Failed to report. err: %+v", err)

21
config/config.go
View File

@@ -1121,18 +1121,17 @@ type ServerInfo struct {
Lockfiles []string `toml:"lockfiles,omitempty" json:"lockfiles,omitempty"` // ie) path/to/package-lock.json
FindLock bool `toml:"findLock,omitempty" json:"findLock,omitempty"`
Type string `toml:"type,omitempty" json:"type,omitempty"` // "pseudo" or ""
WordPress WordPressConf `toml:"wordpress,omitempty" json:"wordpress,omitempty"`
IgnoredJSONKeys []string `toml:"ignoredJSONKeys,omitempty" json:"ignoredJSONKeys,omitempty"`
WordPress WordPressConf `toml:"wordpress,omitempty" json:"wordpress,omitempty"`
// used internal
IPv4Addrs []string `toml:"-" json:"ipv4Addrs,omitempty"`
IPv6Addrs []string `toml:"-" json:"ipv6Addrs,omitempty"`
IPSIdentifiers map[IPS]string `toml:"-" json:"ipsIdentifiers,omitempty"`
LogMsgAnsiColor string `toml:"-" json:"-"` // DebugLog Color
Container Container `toml:"-" json:"-"`
Distro Distro `toml:"-" json:"-"`
Mode ScanMode `toml:"-" json:"-"`
// internal use
IPv4Addrs []string `toml:"-" json:"ipv4Addrs,omitempty"`
IPv6Addrs []string `toml:"-" json:"ipv6Addrs,omitempty"`
IPSIdentifiers map[IPS]string `toml:"-" json:"ipsIdentifiers,omitempty"`
LogMsgAnsiColor string `toml:"-" json:"-"` // DebugLog Color
Container Container `toml:"-" json:"-"`
Distro Distro `toml:"-" json:"-"`
Mode ScanMode `toml:"-" json:"-"`
}
// ContainerSetting is used for loading container setting in config.toml

11
config/tomlloader.go
View File

@@ -45,7 +45,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
d.KeyPassword = keyPass
}
i := 0
index := 0
for serverName, v := range conf.Servers {
if 0 < len(v.KeyPassword) {
return xerrors.Errorf("[Deprecated] KEYPASSWORD IN CONFIG FILE ARE UNSECURE. REMOVE THEM IMMEDIATELY FOR A SECURITY REASONS. THEY WILL BE REMOVED IN A FUTURE RELEASE: %s", serverName)
@@ -268,8 +268,13 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
s.WordPress.OSUser = v.WordPress.OSUser
s.WordPress.IgnoreInactive = v.WordPress.IgnoreInactive
s.LogMsgAnsiColor = Colors[i%len(Colors)]
i++
s.IgnoredJSONKeys = v.IgnoredJSONKeys
if len(s.IgnoredJSONKeys) == 0 {
s.IgnoredJSONKeys = d.IgnoredJSONKeys
}
s.LogMsgAnsiColor = Colors[index%len(Colors)]
index++
servers[serverName] = s
}

21
models/scanresults.go
View File

@@ -3,6 +3,7 @@ package models
import (
"bytes"
"fmt"
"reflect"
"regexp"
"strings"
"time"
@@ -503,3 +504,23 @@ func (r ScanResult) RemoveRaspbianPackFromResult() ScanResult {
return result
}
func (r ScanResult) ClearFields(targetTagNames []string) ScanResult {
if len(targetTagNames) == 0 {
return r
}
target := map[string]bool{}
for _, n := range targetTagNames {
target[strings.ToLower(n)] = true
}
t := reflect.ValueOf(r).Type()
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
jsonValue := strings.Split(f.Tag.Get("json"), ",")[0]
if ok := target[strings.ToLower(jsonValue)]; ok {
vv := reflect.New(f.Type).Elem().Interface()
reflect.ValueOf(&r).Elem().FieldByName(f.Name).Set(reflect.ValueOf(vv))
}
}
return r
}

10
report/localfile.go
View File

@@ -41,14 +41,8 @@ func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) {
}
var b []byte
if c.Conf.Debug {
if b, err = json.MarshalIndent(r, "", " "); err != nil {
return xerrors.Errorf("Failed to Marshal to JSON: %w", err)
}
} else {
if b, err = json.Marshal(r); err != nil {
return xerrors.Errorf("Failed to Marshal to JSON: %w", err)
}
if b, err = json.MarshalIndent(r, "", " "); err != nil {
return xerrors.Errorf("Failed to Marshal to JSON: %w", err)
}
if err := writeFile(p, b, 0600); err != nil {
return xerrors.Errorf("Failed to write JSON. path: %s, err: %w", p, err)

165
report/report.go
View File

@@ -43,105 +43,112 @@ const (
// FillCveInfos fills CVE Detailed Information
func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
var filledResults []models.ScanResult
// Use the same reportedAt for all rs
reportedAt := time.Now()
hostname, _ := os.Hostname()
wpVulnCaches := map[string]string{}
for _, r := range rs {
if c.Conf.RefreshCve || needToRefreshCve(r) {
if !useScannedCves(&r) {
r.ScannedCves = models.VulnInfos{}
}
cpeURIs := []string{}
for i, r := range rs {
if !c.Conf.RefreshCve && !needToRefreshCve(r) {
util.Log.Info("No need to refresh")
continue
}
if len(r.Container.ContainerID) == 0 {
cpeURIs = c.Conf.Servers[r.ServerName].CpeNames
owaspDCXMLPath := c.Conf.Servers[r.ServerName].OwaspDCXMLPath
if owaspDCXMLPath != "" {
cpes, err := parser.Parse(owaspDCXMLPath)
if err != nil {
return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w",
r.ServerName, owaspDCXMLPath, err)
}
cpeURIs = append(cpeURIs, cpes...)
}
} else {
// runningContainer
if s, ok := c.Conf.Servers[r.ServerName]; ok {
if con, ok := s.Containers[r.Container.Name]; ok {
cpeURIs = con.Cpes
owaspDCXMLPath := con.OwaspDCXMLPath
if owaspDCXMLPath != "" {
cpes, err := parser.Parse(owaspDCXMLPath)
if err != nil {
return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w",
r.ServerInfo(), owaspDCXMLPath, err)
}
cpeURIs = append(cpeURIs, cpes...)
}
}
if !useScannedCves(&r) {
r.ScannedCves = models.VulnInfos{}
}
cpeURIs := []string{}
if len(r.Container.ContainerID) == 0 {
cpeURIs = c.Conf.Servers[r.ServerName].CpeNames
owaspDCXMLPath := c.Conf.Servers[r.ServerName].OwaspDCXMLPath
if owaspDCXMLPath != "" {
cpes, err := parser.Parse(owaspDCXMLPath)
if err != nil {
return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w",
r.ServerName, owaspDCXMLPath, err)
}
cpeURIs = append(cpeURIs, cpes...)
}
nCVEs, err := libmanager.DetectLibsCves(&r)
if err != nil {
return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err)
}
util.Log.Infof("%s: %d CVEs are detected with Library",
r.FormatServerName(), nCVEs)
// Integrations
githubInts := GithubSecurityAlerts(c.Conf.Servers[r.ServerName].GitHubRepos)
wpOpt := WordPressOption{c.Conf.Servers[r.ServerName].WordPress.WPVulnDBToken, &wpVulnCaches}
if err := FillCveInfo(dbclient,
&r,
cpeURIs,
true,
githubInts,
wpOpt); err != nil {
return nil, err
}
r.Lang = c.Conf.Lang
r.ReportedAt = reportedAt
r.ReportedVersion = c.Version
r.ReportedRevision = c.Revision
r.ReportedBy = hostname
r.Config.Report = c.Conf
r.Config.Report.Servers = map[string]c.ServerInfo{
r.ServerName: c.Conf.Servers[r.ServerName],
}
if err := overwriteJSONFile(dir, r); err != nil {
return nil, xerrors.Errorf("Failed to write JSON: %w", err)
}
filledResults = append(filledResults, r)
} else {
util.Log.Debugf("No need to refresh")
filledResults = append(filledResults, r)
// runningContainer
if s, ok := c.Conf.Servers[r.ServerName]; ok {
if con, ok := s.Containers[r.Container.Name]; ok {
cpeURIs = con.Cpes
owaspDCXMLPath := con.OwaspDCXMLPath
if owaspDCXMLPath != "" {
cpes, err := parser.Parse(owaspDCXMLPath)
if err != nil {
return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w",
r.ServerInfo(), owaspDCXMLPath, err)
}
cpeURIs = append(cpeURIs, cpes...)
}
}
}
}
nCVEs, err := libmanager.DetectLibsCves(&r)
if err != nil {
return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err)
}
util.Log.Infof("%s: %d CVEs are detected with Library",
r.FormatServerName(), nCVEs)
// Integrations
githubInts := GithubSecurityAlerts(c.Conf.Servers[r.ServerName].GitHubRepos)
wpVulnCaches := map[string]string{}
wpOpt := WordPressOption{c.Conf.Servers[r.ServerName].WordPress.WPVulnDBToken, &wpVulnCaches}
if err := FillCveInfo(dbclient,
&r,
cpeURIs,
true,
githubInts,
wpOpt); err != nil {
return nil, err
}
r.ReportedBy, _ = os.Hostname()
r.Lang = c.Conf.Lang
r.ReportedAt = reportedAt
r.ReportedVersion = c.Version
r.ReportedRevision = c.Revision
r.Config.Report = c.Conf
r.Config.Report.Servers = map[string]c.ServerInfo{
r.ServerName: c.Conf.Servers[r.ServerName],
}
rs[i] = r
}
// Overwrite the json file every time to clear the fields specified in config.IgnoredJSONKeys
for _, r := range rs {
if s, ok := c.Conf.Servers[r.ServerName]; ok {
r = r.ClearFields(s.IgnoredJSONKeys)
}
if err := overwriteJSONFile(dir, r); err != nil {
return nil, xerrors.Errorf("Failed to write JSON: %w", err)
}
}
if c.Conf.Diff {
prevs, err := loadPrevious(filledResults)
prevs, err := loadPrevious(rs)
if err != nil {
return nil, err
}
diff, err := diff(filledResults, prevs)
diff, err := diff(rs, prevs)
if err != nil {
return nil, err
}
filledResults = []models.ScanResult{}
for _, r := range diff {
for i, r := range diff {
if err := fillCvesWithNvdJvn(dbclient.CveDB, &r); err != nil {
return nil, err
}
filledResults = append(filledResults, r)
rs[i] = r
}
}
filtered := []models.ScanResult{}
for _, r := range filledResults {
for i, r := range rs {
r = r.FilterByCvssOver(c.Conf.CvssScoreOver)
r = r.FilterIgnoreCves()
r = r.FilterUnfixed()
@@ -150,9 +157,9 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode
if c.Conf.IgnoreUnscoredCves {
r.ScannedCves = r.ScannedCves.FindScoredVulns()
}
filtered = append(filtered, r)
rs[i] = r
}
return filtered, nil
return rs, nil
}
// FillCveInfo fill scanResult with cve info.