From 0de38b99c265947a344c673b629a0252444ef34e Mon Sep 17 00:00:00 2001 From: kota kanbe Date: Sat, 4 Jun 2016 11:15:53 +0900 Subject: [PATCH] Add -report-s3 option --- README.md | 38 ++++++++++++---- commands/scan.go | 29 ++++++++++++ config/config.go | 5 +++ glide.lock | 31 ++++++++++++- report/s3.go | 112 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 report/s3.go diff --git a/README.md b/README.md index 740b0454..45bdefc0 100644 --- a/README.md +++ b/README.md @@ -486,6 +486,7 @@ scan: [-ignore-unscored-cves] [-report-json] [-report-mail] + [-report-s3] [-report-slack] [-report-text] [-http-proxy=http://192.168.0.1:8080] @@ -493,10 +494,20 @@ scan: [-ask-key-password] [-debug] [-debug-sql] + [-aws-profile=default] + [-aws-region=us-west-2] + [-aws-s3-bucket=bucket_name] + -ask-key-password Ask ssh privatekey password before scanning -ask-sudo-password Ask sudo password of target servers before scanning + -aws-profile string + AWS Profile to use (default "default") + -aws-region string + AWS Region to use (default "us-east-1") + -aws-s3-bucket string + S3 bucket name -config string /path/to/toml (default "$PWD/config.toml") --cve-dictionary-dbpath string @@ -521,6 +532,8 @@ scan: Write report to JSON files ($PWD/results/current) -report-mail Send report via Email + -report-s3 + Write report to S3 (bucket/yyyyMMdd_HHmm) -report-slack Send report via Slack -report-text @@ -546,15 +559,10 @@ scan: | NOPASSWORD | - | defined as NOPASSWORD in /etc/sudoers on target servers | | with password | required | . | -## -report-json option +## -report-json , -report-text option -At the end of the scan, scan results will be available in JSON format in the $PWD/result/current/ directory. -all.json includes the scan results of all servres and servername.json includes the scan result of the server. - -## -report-text option - -At the end of the scan, scan results will be available in TEXT format in the $PWD/result/current/ directory. -all.txt includes the scan results of all servres and servername.txt includes the scan result of the server. +At the end of the scan, scan results will be available in the $PWD/result/current/ directory. +all.(json|txt) includes the scan results of all servres and servername.(json|txt) includes the scan result of the server. ## example @@ -579,6 +587,20 @@ With this sample command, it will .. - Scan only 2 servers (server1, server2) - Print scan result to terminal +### Put results in S3 bucket +To put results in S3 bucket, configure following settings in AWS before scanning. +- Create S3 bucket. see [Creating a Bucket](http://docs.aws.amazon.com/AmazonS3/latest/UG/CreatingaBucket.html) +- Create access key. The access key must have read and write access to the AWS S3 bucket. see [Managing Access Keys for IAM Users](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) +- Configure the security credentials. see [Configuring the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) + +``` +$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 -aws-region=ap-northeast-1 -aws-s3-bucket=vuls -aws-profile=default +``` +With this sample command, it will .. +- Use SSH Key-Based authentication with empty password (without -ask-key-password option) +- Sudo with no password (without -ask-sudo-password option) +- Scan all servers defined in config file +- Put scan result(JSON) in S3 bucket. The bucket name is "vuls" in ap-northeast-1 and profile is "default" ---- diff --git a/commands/scan.go b/commands/scan.go index f9cdf1d5..17f0b7ac 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -56,12 +56,17 @@ type ScanCmd struct { reportMail bool reportJSON bool reportText bool + reportS3 bool askSudoPassword bool askKeyPassword bool useYumPluginSecurity bool useUnattendedUpgrades bool + + awsProfile string + awsS3Bucket string + awsRegion string } // Name return subcommand name @@ -83,6 +88,7 @@ func (*ScanCmd) Usage() string { [-ignore-unscored-cves] [-report-json] [-report-mail] + [-report-s3] [-report-slack] [-report-text] [-http-proxy=http://192.168.0.1:8080] @@ -90,6 +96,9 @@ func (*ScanCmd) Usage() string { [-ask-key-password] [-debug] [-debug-sql] + [-aws-profile=default] + [-aws-region=us-west-2] + [-aws-s3-bucket=bucket_name] ` } @@ -152,6 +161,15 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { fmt.Sprintf("Write report to text files (%s/results/current)", wd), ) + f.BoolVar(&p.reportS3, + "report-s3", + false, + "Write report to S3 (bucket/yyyyMMdd_HHmm)", + ) + f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS Profile to use") + f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS Region to use") + f.StringVar(&p.awsS3Bucket, "aws-s3-bucket", "", "S3 bucket name") + f.BoolVar( &p.askKeyPassword, "ask-key-password", @@ -257,6 +275,17 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if p.reportText { reports = append(reports, report.TextFileWriter{}) } + if p.reportS3 { + c.Conf.AwsRegion = p.awsRegion + c.Conf.AwsProfile = p.awsProfile + c.Conf.S3Bucket = p.awsS3Bucket + if err := report.CheckIfBucketExists(); err != nil { + Log.Errorf("Failed to access to the S3 bucket. err: %s", err) + Log.Error("Ensure the bucket or check AWS config before scanning") + return subcommands.ExitUsageError + } + reports = append(reports, report.S3Writer{}) + } c.Conf.DBPath = p.dbpath c.Conf.CveDBPath = p.cvedbpath diff --git a/config/config.go b/config/config.go index fcbf2787..292e45c5 100644 --- a/config/config.go +++ b/config/config.go @@ -47,6 +47,11 @@ type Config struct { HTTPProxy string `valid:"url"` DBPath string CveDBPath string + + AwsProfile string + AwsRegion string + S3Bucket string + // CpeNames []string // SummaryMode bool UseYumPluginSecurity bool diff --git a/glide.lock b/glide.lock index cf7cc9d8..c8df0fe7 100644 --- a/glide.lock +++ b/glide.lock @@ -1,14 +1,41 @@ hash: f6451157cbeaa3590d2b62fb9902221f73c81b5aeda722cf61929f88ce1cfc64 -updated: 2016-06-02T14:32:59.414338172+09:00 +updated: 2016-06-06T09:28:24.182725693+09:00 imports: - name: github.com/asaskevich/govalidator version: df81827fdd59d8b4fb93d8910b286ab7a3919520 +- name: github.com/aws/aws-sdk-go + version: 1608a1f6ffe7f327007d2aca50ebf1d26c285ef2 + subpackages: + - aws + - aws/credentials + - aws/session + - service/s3 + - aws/awserr + - aws/client + - aws/corehandlers + - aws/defaults + - aws/request + - private/endpoints + - aws/awsutil + - aws/client/metadata + - private/protocol + - private/protocol/restxml + - private/signer/v4 + - private/waiter + - aws/credentials/ec2rolecreds + - aws/ec2metadata + - private/protocol/query + - private/protocol/rest + - private/protocol/xml/xmlutil + - private/protocol/query/queryutil - name: github.com/BurntSushi/toml version: f0aeabca5a127c4078abb8c8d64298b147264b55 - name: github.com/cenkalti/backoff version: a6030178a585d5972d4d33ce61f4a1fa40eaaed0 - name: github.com/cheggaaa/pb version: c1f48d5ce4f292dfb775ef52aaedd15be323510d +- name: github.com/go-ini/ini + version: 72ba3e6b9e6b87e0c74c9a7a4dc86e8dd8ba4355 - name: github.com/google/subcommands version: 1c7173745a6001f67d8d96ab4e178284c77f7759 - name: github.com/gosuri/uitable @@ -22,6 +49,8 @@ imports: version: bf0e23607840019d62e32e87e9af48e881d74b1e - name: github.com/jinzhu/inflection version: 3272df6c21d04180007eb3349844c89a3856bc25 +- name: github.com/jmespath/go-jmespath + version: 0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74 - name: github.com/jroimartin/gocui version: 2dcda558bf18ec07c7065bf1eaf071b5305f7c0c - name: github.com/k0kubun/pp diff --git a/report/s3.go b/report/s3.go new file mode 100644 index 00000000..1338b093 --- /dev/null +++ b/report/s3.go @@ -0,0 +1,112 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package report + +import ( + "bytes" + "encoding/json" + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + + c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" +) + +// S3Writer writes results to S3 +type S3Writer struct{} + +func getS3() *s3.S3 { + return s3.New(session.New(&aws.Config{ + Region: aws.String(c.Conf.AwsRegion), + Credentials: credentials.NewSharedCredentials("", c.Conf.AwsProfile), + })) +} + +// CheckIfBucketExists check the existence of S3 bucket +func CheckIfBucketExists() error { + svc := getS3() + result, err := svc.ListBuckets(&s3.ListBucketsInput{}) + if err != nil { + return fmt.Errorf( + "Failed to list buckets. err: %s, profile: %s, region: %s", + err, c.Conf.AwsProfile, c.Conf.AwsRegion) + } + + found := false + for _, bucket := range result.Buckets { + if *bucket.Name == c.Conf.S3Bucket { + found = true + break + } + } + if !found { + return fmt.Errorf( + "Failed to find the buckets. profile: %s, region: %s, bukdet: %s", + c.Conf.AwsProfile, c.Conf.AwsRegion, c.Conf.S3Bucket) + } + return nil +} + +// Write put results in S3 +func (w S3Writer) Write(scanResults []models.ScanResult) (err error) { + + var jsonBytes []byte + if jsonBytes, err = json.MarshalIndent(scanResults, "", " "); err != nil { + return fmt.Errorf("Failed to Marshal to JSON: %s", err) + } + + // http://docs.aws.amazon.com/sdk-for-go/latest/v1/developerguide/common-examples.title.html + svc := getS3() + timestr := time.Now().Format("20060102_1504") + key := fmt.Sprintf("%s/%s", timestr, "all.json") + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: &c.Conf.S3Bucket, + Key: &key, + Body: bytes.NewReader(jsonBytes), + }) + if err != nil { + return fmt.Errorf("Failed to upload data to %s/%s, %s", c.Conf.S3Bucket, key, err) + } + + for _, r := range scanResults { + key := "" + if r.Container.ContainerID == "" { + key = fmt.Sprintf("%s/%s.json", timestr, r.ServerName) + } else { + key = fmt.Sprintf("%s/%s_%s.json", timestr, r.ServerName, r.Container.Name) + } + + if jsonBytes, err = json.MarshalIndent(r, "", " "); err != nil { + return fmt.Errorf("Failed to Marshal to JSON: %s", err) + } + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: &c.Conf.S3Bucket, + Key: &key, + Body: bytes.NewReader(jsonBytes), + }) + if err != nil { + return fmt.Errorf("Failed to upload data to %s/%s, %s", c.Conf.S3Bucket, key, err) + } + } + return nil +}