Merge pull request #92 from future-architect/s3-output

Add -report-s3 option
2016-06-06 09:31:41 +09:00
parent 1044fb8574 0de38b99c2
commit 259f23f6ee
5 changed files with 206 additions and 9 deletions
--- a/README.md
+++ b/README.md
@@ -486,6 +486,7 @@ scan:
                [-ignore-unscored-cves]
                [-report-json]
                [-report-mail]
+                [-report-s3]
                [-report-slack]
                [-report-text]
                [-http-proxy=http://192.168.0.1:8080]
@@ -493,10 +494,20 @@ scan:
                [-ask-key-password]
                [-debug]
                [-debug-sql]
+                [-aws-profile=default]
+                [-aws-region=us-west-2]
+                [-aws-s3-bucket=bucket_name]
+
  -ask-key-password
        Ask ssh privatekey password before scanning
  -ask-sudo-password
        Ask sudo password of target servers before scanning
+  -aws-profile string
+        AWS Profile to use (default "default")
+  -aws-region string
+        AWS Region to use (default "us-east-1")
+  -aws-s3-bucket string
+        S3 bucket name
  -config string
        /path/to/toml (default "$PWD/config.toml")
  --cve-dictionary-dbpath string
@@ -521,6 +532,8 @@ scan:
        Write report to JSON files ($PWD/results/current)
  -report-mail
        Send report via Email
+  -report-s3
+        Write report to S3 (bucket/yyyyMMdd_HHmm)
  -report-slack
        Send report via Slack
  -report-text
@@ -546,15 +559,10 @@ scan:
 | NOPASSWORD       | - | defined as NOPASSWORD in /etc/sudoers on target servers |
 | with password    | required | . |

-## -report-json option
+## -report-json , -report-text option

-At the end of the scan, scan results will be available in JSON format in the $PWD/result/current/ directory.  
-all.json includes the scan results of all servres and servername.json includes the scan result of the server.
-
-## -report-text option
-
-At the end of the scan, scan results will be available in TEXT format in the $PWD/result/current/ directory.  
-all.txt includes the scan results of all servres and servername.txt includes the scan result of the server.
+At the end of the scan, scan results will be available in the $PWD/result/current/ directory.  
+all.(json|txt) includes the scan results of all servres and servername.(json|txt) includes the scan result of the server.

 ## example

@@ -579,6 +587,20 @@ With this sample command, it will ..
 - Scan only 2 servers (server1, server2)
 - Print scan result to terminal

+### Put results in S3 bucket
+To put results in S3 bucket, configure following settings in AWS before scanning.
+- Create S3 bucket. see [Creating a Bucket](http://docs.aws.amazon.com/AmazonS3/latest/UG/CreatingaBucket.html)  
+- Create access key. The access key must have read and write access to the AWS S3 bucket. see [Managing Access Keys for IAM Users](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
+- Configure the security credentials. see [Configuring the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html)
+
+```
+$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 -aws-region=ap-northeast-1 -aws-s3-bucket=vuls -aws-profile=default
+```
+With this sample command, it will ..
+- Use SSH Key-Based authentication with empty password (without -ask-key-password option)
+- Sudo with no password (without -ask-sudo-password option)
+- Scan all servers defined in config file
+- Put scan result(JSON) in S3 bucket. The bucket name is "vuls" in ap-northeast-1 and profile is "default"


 ----
--- a/commands/scan.go
+++ b/commands/scan.go
@@ -56,12 +56,17 @@ type ScanCmd struct {
 	reportMail  bool
 	reportJSON  bool
 	reportText  bool
+	reportS3    bool

 	askSudoPassword bool
 	askKeyPassword  bool

 	useYumPluginSecurity  bool
 	useUnattendedUpgrades bool
+
+	awsProfile  string
+	awsS3Bucket string
+	awsRegion   string
 }

 // Name return subcommand name
@@ -83,6 +88,7 @@ func (*ScanCmd) Usage() string {
 		[-ignore-unscored-cves]
 		[-report-json]
 		[-report-mail]
+		[-report-s3]
 		[-report-slack]
 		[-report-text]
 		[-http-proxy=http://192.168.0.1:8080]
@@ -90,6 +96,9 @@ func (*ScanCmd) Usage() string {
 		[-ask-key-password]
 		[-debug]
 		[-debug-sql]
+		[-aws-profile=default]
+		[-aws-region=us-west-2]
+		[-aws-s3-bucket=bucket_name]
 `
 }

@@ -152,6 +161,15 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
 		fmt.Sprintf("Write report to text files (%s/results/current)", wd),
 	)

+	f.BoolVar(&p.reportS3,
+		"report-s3",
+		false,
+		"Write report to S3 (bucket/yyyyMMdd_HHmm)",
+	)
+	f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS Profile to use")
+	f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS Region to use")
+	f.StringVar(&p.awsS3Bucket, "aws-s3-bucket", "", "S3 bucket name")
+
 	f.BoolVar(
 		&p.askKeyPassword,
 		"ask-key-password",
@@ -257,6 +275,17 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 	if p.reportText {
 		reports = append(reports, report.TextFileWriter{})
 	}
+	if p.reportS3 {
+		c.Conf.AwsRegion = p.awsRegion
+		c.Conf.AwsProfile = p.awsProfile
+		c.Conf.S3Bucket = p.awsS3Bucket
+		if err := report.CheckIfBucketExists(); err != nil {
+			Log.Errorf("Failed to access to the S3 bucket. err: %s", err)
+			Log.Error("Ensure the bucket or check AWS config before scanning")
+			return subcommands.ExitUsageError
+		}
+		reports = append(reports, report.S3Writer{})
+	}

 	c.Conf.DBPath = p.dbpath
 	c.Conf.CveDBPath = p.cvedbpath
--- a/config/config.go
+++ b/config/config.go
@@ -47,6 +47,11 @@ type Config struct {
 	HTTPProxy string `valid:"url"`
 	DBPath    string
 	CveDBPath string
+
+	AwsProfile string
+	AwsRegion  string
+	S3Bucket   string
+
 	//  CpeNames      []string
 	//  SummaryMode          bool
 	UseYumPluginSecurity  bool
--- a/glide.lock
+++ b/glide.lock
@@ -1,14 +1,41 @@
 hash: f6451157cbeaa3590d2b62fb9902221f73c81b5aeda722cf61929f88ce1cfc64
-updated: 2016-06-02T14:32:59.414338172+09:00
+updated: 2016-06-06T09:28:24.182725693+09:00
 imports:
 - name: github.com/asaskevich/govalidator
  version: df81827fdd59d8b4fb93d8910b286ab7a3919520
+- name: github.com/aws/aws-sdk-go
+  version: 1608a1f6ffe7f327007d2aca50ebf1d26c285ef2
+  subpackages:
+  - aws
+  - aws/credentials
+  - aws/session
+  - service/s3
+  - aws/awserr
+  - aws/client
+  - aws/corehandlers
+  - aws/defaults
+  - aws/request
+  - private/endpoints
+  - aws/awsutil
+  - aws/client/metadata
+  - private/protocol
+  - private/protocol/restxml
+  - private/signer/v4
+  - private/waiter
+  - aws/credentials/ec2rolecreds
+  - aws/ec2metadata
+  - private/protocol/query
+  - private/protocol/rest
+  - private/protocol/xml/xmlutil
+  - private/protocol/query/queryutil
 - name: github.com/BurntSushi/toml
  version: f0aeabca5a127c4078abb8c8d64298b147264b55
 - name: github.com/cenkalti/backoff
  version: a6030178a585d5972d4d33ce61f4a1fa40eaaed0
 - name: github.com/cheggaaa/pb
  version: c1f48d5ce4f292dfb775ef52aaedd15be323510d
+- name: github.com/go-ini/ini
+  version: 72ba3e6b9e6b87e0c74c9a7a4dc86e8dd8ba4355
 - name: github.com/google/subcommands
  version: 1c7173745a6001f67d8d96ab4e178284c77f7759
 - name: github.com/gosuri/uitable
@@ -22,6 +49,8 @@ imports:
  version: bf0e23607840019d62e32e87e9af48e881d74b1e
 - name: github.com/jinzhu/inflection
  version: 3272df6c21d04180007eb3349844c89a3856bc25
+- name: github.com/jmespath/go-jmespath
+  version: 0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74
 - name: github.com/jroimartin/gocui
  version: 2dcda558bf18ec07c7065bf1eaf071b5305f7c0c
 - name: github.com/k0kubun/pp
--- a/report/s3.go
+++ b/report/s3.go
@@ -0,0 +1,112 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package report
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+
+	c "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+)
+
+// S3Writer writes results to S3
+type S3Writer struct{}
+
+func getS3() *s3.S3 {
+	return s3.New(session.New(&aws.Config{
+		Region:      aws.String(c.Conf.AwsRegion),
+		Credentials: credentials.NewSharedCredentials("", c.Conf.AwsProfile),
+	}))
+}
+
+// CheckIfBucketExists check the existence of S3 bucket
+func CheckIfBucketExists() error {
+	svc := getS3()
+	result, err := svc.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		return fmt.Errorf(
+			"Failed to list buckets. err: %s, profile: %s, region: %s",
+			err, c.Conf.AwsProfile, c.Conf.AwsRegion)
+	}
+
+	found := false
+	for _, bucket := range result.Buckets {
+		if *bucket.Name == c.Conf.S3Bucket {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return fmt.Errorf(
+			"Failed to find the buckets. profile: %s, region: %s, bukdet: %s",
+			c.Conf.AwsProfile, c.Conf.AwsRegion, c.Conf.S3Bucket)
+	}
+	return nil
+}
+
+// Write put results in S3
+func (w S3Writer) Write(scanResults []models.ScanResult) (err error) {
+
+	var jsonBytes []byte
+	if jsonBytes, err = json.MarshalIndent(scanResults, "", "  "); err != nil {
+		return fmt.Errorf("Failed to Marshal to JSON: %s", err)
+	}
+
+	// http://docs.aws.amazon.com/sdk-for-go/latest/v1/developerguide/common-examples.title.html
+	svc := getS3()
+	timestr := time.Now().Format("20060102_1504")
+	key := fmt.Sprintf("%s/%s", timestr, "all.json")
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket: &c.Conf.S3Bucket,
+		Key:    &key,
+		Body:   bytes.NewReader(jsonBytes),
+	})
+	if err != nil {
+		return fmt.Errorf("Failed to upload data to %s/%s, %s", c.Conf.S3Bucket, key, err)
+	}
+
+	for _, r := range scanResults {
+		key := ""
+		if r.Container.ContainerID == "" {
+			key = fmt.Sprintf("%s/%s.json", timestr, r.ServerName)
+		} else {
+			key = fmt.Sprintf("%s/%s_%s.json", timestr, r.ServerName, r.Container.Name)
+		}
+
+		if jsonBytes, err = json.MarshalIndent(r, "", "  "); err != nil {
+			return fmt.Errorf("Failed to Marshal to JSON: %s", err)
+		}
+		_, err = svc.PutObject(&s3.PutObjectInput{
+			Bucket: &c.Conf.S3Bucket,
+			Key:    &key,
+			Body:   bytes.NewReader(jsonBytes),
+		})
+		if err != nil {
+			return fmt.Errorf("Failed to upload data to %s/%s, %s", c.Conf.S3Bucket, key, err)
+		}
+	}
+	return nil
+}