diff --git a/README.ja.md b/README.ja.md index 11d5eea5..6d93cbad 100644 --- a/README.ja.md +++ b/README.ja.md @@ -467,6 +467,7 @@ prepare [-config=/path/to/config.toml] [-debug] [-ask-sudo-password] [-ask-key-password] + [SERVER]... -ask-key-password Ask ssh privatekey password before scanning @@ -497,6 +498,7 @@ scan: [-cvss-over=7] [-ignore-unscored-cves] [-ssh-external] + [-report-azure-blob] [-report-json] [-report-mail] [-report-s3] @@ -510,6 +512,12 @@ scan: [-aws-profile=default] [-aws-region=us-west-2] [-aws-s3-bucket=bucket_name] + [-azure-account=accout] + [-azure-key=key] + [-azure-container=container] + [SERVER]... + + -ask-key-password Ask ssh privatekey password before scanning @@ -521,9 +529,15 @@ scan: AWS Region to use (default "us-east-1") -aws-s3-bucket string S3 bucket name + -azure-account string + Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified + -azure-container string + Azure storage container name + -azure-key string + Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified -config string /path/to/toml (default "$PWD/config.toml") - --cve-dictionary-dbpath string + -cve-dictionary-dbpath string /path/to/sqlite3 (For get cve detail from cve.sqlite3) -cve-dictionary-url string http://CVE.Dictionary (default "http://127.0.0.1:1323") @@ -591,14 +605,12 @@ SSH Configが使えるので、ProxyCommandを使った多段SSHなどが可能 `all.(json|txt)`には、全サーバのスキャン結果が出力される。 `servername.(json|txt)`には、サーバごとのスキャン結果が出力される。 -## example - -### Scan all servers defined in config file +## Example: Scan all servers defined in config file ``` $ vuls scan \ - --report-slack \ - --report-mail \ - --cvss-over=7 \ + -report-slack \ + -report-mail \ + -cvss-over=7 \ -ask-sudo-password \ -ask-key-password \ -cve-dictionary-dbpath=$PWD/cve.sqlite3 @@ -611,7 +623,7 @@ $ vuls scan \ - CVSSスコアが 7.0 以上の脆弱性のみレポート - go-cve-dictionaryにはHTTPではなくDBに直接アクセス（go-cve-dictionaryをサーバモードで起動しない） -### Scan specific servers +## Example: Scan specific servers ``` $ vuls scan \ -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ @@ -622,8 +634,7 @@ $ vuls scan \ - ノーパスワードでsudoが実行可能 - configで定義されているサーバの中の、server1, server2のみスキャン -### Put results in S3 bucket -レポートをS3バケットに格納する方法 +## Example: Put results in S3 bucket 事前にAWS関連の設定を行う - S3バケットを作成 [Creating a Bucket](http://docs.aws.amazon.com/AmazonS3/latest/UG/CreatingaBucket.html) @@ -633,6 +644,7 @@ $ vuls scan \ ``` $ vuls scan \ -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ + -report-s3 -aws-region=ap-northeast-1 \ -aws-s3-bucket=vuls \ -aws-profile=default @@ -646,6 +658,38 @@ $ vuls scan \ - リージョン ... ap-northeast-1 - 利用するProfile ... default +## Example: Put results in Azure Blob storage + +事前にAzure Blob関連の設定を行う +- Containerを作成 + +``` +$ vuls scan \ + -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ + -report-azure-blob \ + -azure-container=vuls \ + -azure-account=test \ + -azure-key=access-key-string +``` +この例では、 +- SSH公開鍵認証（秘密鍵パスフレーズなし） +- ノーパスワードでsudoが実行可能 +- configに定義された全サーバをスキャン +- 結果をJSON形式でAzure Blobに格納する。 + - コンテナ名 ... vuls + - ストレージアカウント名 ... test + - アクセスキー ... access-key-string + +また、アカウント名とアクセスキーは環境変数でも定義が可能 +``` +$ export AZURE_STORAGE_ACCOUNT=test +$ export AZURE_STORAGE_ACCESS_KEY=access-key-string +$ vuls scan \ + -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ + -report-azure-blob \ + -azure-container=vuls +``` + ---- # Usage: Scan vulnerability of non-OS package diff --git a/README.md b/README.md index 461b2503..71e3b33b 100644 --- a/README.md +++ b/README.md @@ -484,6 +484,7 @@ prepare [-config=/path/to/config.toml] [-debug] [-ask-sudo-password] [-ask-key-password] + [SERVER]... -ask-key-password Ask ssh privatekey password before scanning @@ -514,6 +515,7 @@ scan: [-cvss-over=7] [-ignore-unscored-cves] [-ssh-external] + [-report-azure-blob] [-report-json] [-report-mail] [-report-s3] @@ -527,6 +529,11 @@ scan: [-aws-profile=default] [-aws-region=us-west-2] [-aws-s3-bucket=bucket_name] + [-azure-account=accout] + [-azure-key=key] + [-azure-container=container] + [SERVER]... + -ask-key-password Ask ssh privatekey password before scanning @@ -538,9 +545,15 @@ scan: AWS Region to use (default "us-east-1") -aws-s3-bucket string S3 bucket name - -config string + -azure-account string + Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified + -azure-container string + Azure storage container name + -azure-key string + Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified + -config string /path/to/toml (default "$PWD/config.toml") - --cve-dictionary-dbpath string + -cve-dictionary-dbpath string /path/to/sqlite3 (For get cve detail from cve.sqlite3) -cve-dictionary-url string http://CVE.Dictionary (default "http://127.0.0.1:1323") @@ -607,9 +620,7 @@ This is useful If you want to use ProxyCommand or chiper algorithm of SSH that i At the end of the scan, scan results will be available in the `$PWD/result/current/` directory. `all.(json|txt)` includes the scan results of all servres and `servername.(json|txt)` includes the scan result of the server. -## example - -### Scan all servers defined in config file +## Example: Scan all servers defined in config file ``` $ vuls scan \ --report-slack \ @@ -626,7 +637,7 @@ With this sample command, it will .. - Only Report CVEs that CVSS score is over 7 - Print scan result to terminal -### Scan specific servers +## Example: Scan specific servers ``` $ vuls scan \ -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ @@ -638,7 +649,7 @@ With this sample command, it will .. - Scan only 2 servers (server1, server2) - Print scan result to terminal -### Put results in S3 bucket +## Example: Put results in S3 bucket To put results in S3 bucket, configure following settings in AWS before scanning. - Create S3 bucket. see [Creating a Bucket](http://docs.aws.amazon.com/AmazonS3/latest/UG/CreatingaBucket.html) - Create access key. The access key must have read and write access to the AWS S3 bucket. see [Managing Access Keys for IAM Users](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) @@ -647,6 +658,7 @@ To put results in S3 bucket, configure following settings in AWS before scanning ``` $ vuls scan \ -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ + -report-s3 \ -aws-region=ap-northeast-1 \ -aws-s3-bucket=vuls \ -aws-profile=default @@ -657,6 +669,35 @@ With this sample command, it will .. - Scan all servers defined in config file - Put scan result(JSON) in S3 bucket. The bucket name is "vuls" in ap-northeast-1 and profile is "default" +## Example: Put results in Azure Blob storage + +To put results in Azure Blob Storage, configure following settings in Azure before scanning. +- Create a container + +``` +$ vuls scan \ + -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ + -report-azure-blob \ + -azure-container=vuls \ + -azure-account=test \ + -azure-key=access-key-string +``` +With this sample command, it will .. +- Use SSH Key-Based authentication with empty password (without -ask-key-password option) +- Sudo with no password (without -ask-sudo-password option) +- Scan all servers defined in config file +- Put scan result(JSON) in Azure Blob Storage. The container name is "vuls", storage account is "test" and accesskey is "access-key-string" + +account and access key can be defined in environment variables. +``` +$ export AZURE_STORAGE_ACCOUNT=test +$ export AZURE_STORAGE_ACCESS_KEY=access-key-string +$ vuls scan \ + -cve-dictionary-dbpath=$PWD/cve.sqlite3 \ + -report-azure-blob \ + -azure-container=vuls +``` + ---- diff --git a/commands/prepare.go b/commands/prepare.go index 76a346c4..a0c70146 100644 --- a/commands/prepare.go +++ b/commands/prepare.go @@ -65,6 +65,7 @@ func (*PrepareCmd) Usage() string { [-ask-key-password] [-debug] + [SERVER]... ` } diff --git a/commands/scan.go b/commands/scan.go index ae0cd62b..a737b764 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -49,25 +49,29 @@ type ScanCmd struct { cvssScoreOver float64 ignoreUnscoredCves bool - httpProxy string - - // reporting - reportSlack bool - reportMail bool - reportJSON bool - reportText bool - reportS3 bool - + httpProxy string askSudoPassword bool askKeyPassword bool - useYumPluginSecurity bool - useUnattendedUpgrades bool + // reporting + reportSlack bool + reportMail bool + reportJSON bool + reportText bool + reportS3 bool + reportAzureBlob bool awsProfile string awsS3Bucket string awsRegion string + azureAccount string + azureKey string + azureContainer string + + useYumPluginSecurity bool + useUnattendedUpgrades bool + sshExternal bool } @@ -81,7 +85,6 @@ func (*ScanCmd) Synopsis() string { return "Scan vulnerabilities" } func (*ScanCmd) Usage() string { return `scan: scan - [SERVER]... [-lang=en|ja] [-config=/path/to/config.toml] [-dbpath=/path/to/vuls.sqlite3] @@ -90,6 +93,7 @@ func (*ScanCmd) Usage() string { [-cvss-over=7] [-ignore-unscored-cves] [-ssh-external] + [-report-azure-blob] [-report-json] [-report-mail] [-report-s3] @@ -103,6 +107,13 @@ func (*ScanCmd) Usage() string { [-aws-profile=default] [-aws-region=us-west-2] [-aws-s3-bucket=bucket_name] + [-azure-profile=default] + [-aws-region=us-west-2] + [-azure-account=accout] + [-azure-key=key] + [-azure-container=container] + + [SERVER]... ` } @@ -174,12 +185,21 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.reportS3, "report-s3", false, - "Write report to S3 (bucket/yyyyMMdd_HHmm)", + "Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json)", ) - f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS Profile to use") - f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS Region to use") + f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS profile to use") + f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS region to use") f.StringVar(&p.awsS3Bucket, "aws-s3-bucket", "", "S3 bucket name") + f.BoolVar(&p.reportAzureBlob, + "report-azure-blob", + false, + "Write report to S3 (container/yyyyMMdd_HHmm/servername.json)", + ) + f.StringVar(&p.azureAccount, "azure-account", "", "Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified") + f.StringVar(&p.azureKey, "azure-key", "", "Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified") + f.StringVar(&p.azureContainer, "azure-container", "", "Azure storage container name") + f.BoolVar( &p.askKeyPassword, "ask-key-password", @@ -296,6 +316,29 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } reports = append(reports, report.S3Writer{}) } + if p.reportAzureBlob { + c.Conf.AzureAccount = p.azureAccount + if c.Conf.AzureAccount == "" { + c.Conf.AzureAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") + } + + c.Conf.AzureKey = p.azureKey + if c.Conf.AzureKey == "" { + c.Conf.AzureKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY") + } + + c.Conf.AzureContainer = p.azureContainer + if c.Conf.AzureContainer == "" { + Log.Error("Azure storage container name is requied with --azure-container option") + return subcommands.ExitUsageError + } + if err := report.CheckIfAzureContainerExists(); err != nil { + Log.Errorf("Failed to access to the Azure Blob container. err: %s", err) + Log.Error("Ensure the container or check Azure config before scanning") + return subcommands.ExitUsageError + } + reports = append(reports, report.AzureBlobWriter{}) + } c.Conf.DBPath = p.dbpath c.Conf.CveDBPath = p.cvedbpath diff --git a/config/config.go b/config/config.go index 057696a7..220398fb 100644 --- a/config/config.go +++ b/config/config.go @@ -54,6 +54,10 @@ type Config struct { AwsRegion string S3Bucket string + AzureAccount string + AzureKey string + AzureContainer string + // CpeNames []string // SummaryMode bool UseYumPluginSecurity bool diff --git a/glide.lock b/glide.lock index 56558fdd..282caddd 100644 --- a/glide.lock +++ b/glide.lock @@ -1,10 +1,10 @@ -hash: f6451157cbeaa3590d2b62fb9902221f73c81b5aeda722cf61929f88ce1cfc64 -updated: 2016-07-03T16:13:02.040590043+09:00 +hash: 9683c87b3cf998e7fac1b12c4a94bf2bd18cb5422e9108539811546e703a439a +updated: 2016-07-12T16:20:45.462913061+09:00 imports: - name: github.com/asaskevich/govalidator version: df81827fdd59d8b4fb93d8910b286ab7a3919520 - name: github.com/aws/aws-sdk-go - version: caee6e866bf437a6bef0777a3bf141cdd3aa022d + version: 90dec2183a5f5458ee79cbaf4b8e9ab910bc81a6 subpackages: - aws - aws/credentials @@ -28,14 +28,18 @@ imports: - private/protocol/query - private/protocol/xml/xmlutil - private/protocol/query/queryutil +- name: github.com/Azure/azure-sdk-for-go + version: 58a13e378daf3b06e65925397185684b16321111 + subpackages: + - storage - name: github.com/BurntSushi/toml - version: f0aeabca5a127c4078abb8c8d64298b147264b55 + version: ffaa107fbd880f6d18cd6fec9b511668dcad8639 - name: github.com/cenkalti/backoff version: cdf48bbc1eb78d1349cbda326a4a037f7ba565c6 - name: github.com/cheggaaa/pb - version: c1f48d5ce4f292dfb775ef52aaedd15be323510d + version: 04b234c80d661c663dbcebd52fc7218fdacc6d0c - name: github.com/go-ini/ini - version: 927d8d7ced542ab92df77ac1637b6e56336ee0dd + version: cf53f9204df4fbdd7ec4164b57fa6184ba168292 - name: github.com/google/subcommands version: 1c7173745a6001f67d8d96ab4e178284c77f7759 - name: github.com/gosuri/uitable @@ -46,7 +50,7 @@ imports: - name: github.com/howeyc/gopass version: 66487b23f2880ba32e185121d2cd51a338ea069a - name: github.com/jinzhu/gorm - version: c1c4f9f86e732a042aac9f37e025893d6d6cabec + version: 613c0655691abb7691b70c5fda80a716d9e20b1b - name: github.com/jinzhu/inflection version: 8f4d3a0d04ce0b7c0cf3126fb98524246d00d102 - name: github.com/jmespath/go-jmespath @@ -89,7 +93,7 @@ imports: - name: github.com/Sirupsen/logrus version: f3cfb454f4c209e6668c95216c4744b8fddb2356 - name: golang.org/x/crypto - version: 811831de4c4dd03a0b8737233af3b36852386373 + version: c2f4947f41766b144bb09066e919466da5eddeae subpackages: - ssh - ssh/agent @@ -98,12 +102,12 @@ imports: - ed25519 - ed25519/internal/edwards25519 - name: golang.org/x/net - version: b400c2eff1badec7022a8c8f5bea058b6315eed7 + version: f841c39de738b1d0df95b5a7187744f0e03d8112 subpackages: - context - publicsuffix - name: golang.org/x/sys - version: 62bee037599929a6e9146f29d10dd5208c43507d + version: a408501be4d17ee978c04a618e7a1b22af058c0e subpackages: - unix - name: gopkg.in/alexcesaro/quotedprintable.v3 diff --git a/glide.yaml b/glide.yaml index 495d831e..9d328dd9 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,8 +1,17 @@ package: github.com/future-architect/vuls import: +- package: github.com/Azure/azure-sdk-for-go + subpackages: + - storage - package: github.com/BurntSushi/toml - package: github.com/Sirupsen/logrus - package: github.com/asaskevich/govalidator +- package: github.com/aws/aws-sdk-go + subpackages: + - aws + - aws/credentials + - aws/session + - service/s3 - package: github.com/cenkalti/backoff - package: github.com/google/subcommands - package: github.com/gosuri/uitable diff --git a/report/azureblob.go b/report/azureblob.go new file mode 100644 index 00000000..e44187f0 --- /dev/null +++ b/report/azureblob.go @@ -0,0 +1,140 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package report + +import ( + "bytes" + "encoding/json" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/storage" + + c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" +) + +// AzureBlobWriter writes results to AzureBlob +type AzureBlobWriter struct{} + +// CheckIfAzureContainerExists check the existence of Azure storage container +func CheckIfAzureContainerExists() error { + cli, err := getBlobClient() + if err != nil { + return err + } + ok, err := cli.ContainerExists(c.Conf.AzureContainer) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("Container not found. Container: %s", c.Conf.AzureContainer) + } + return nil +} + +func getBlobClient() (storage.BlobStorageClient, error) { + api, err := storage.NewBasicClient(c.Conf.AzureAccount, c.Conf.AzureKey) + if err != nil { + return storage.BlobStorageClient{}, err + } + return api.GetBlobService(), nil +} + +// Write results to Azure Blob storage +func (w AzureBlobWriter) Write(scanResults []models.ScanResult) (err error) { + reqChan := make(chan models.ScanResult, len(scanResults)) + resChan := make(chan bool) + errChan := make(chan error, len(scanResults)) + defer close(resChan) + defer close(errChan) + defer close(reqChan) + + timeout := time.After(10 * 60 * time.Second) + concurrency := 10 + tasks := util.GenWorkers(concurrency) + + go func() { + for _, r := range scanResults { + reqChan <- r + } + }() + + for range scanResults { + tasks <- func() { + select { + case sresult := <-reqChan: + func(r models.ScanResult) { + err := w.upload(r) + if err != nil { + errChan <- err + } + resChan <- true + }(sresult) + } + } + } + + errs := []error{} + for i := 0; i < len(scanResults); i++ { + select { + case <-resChan: + case err := <-errChan: + errs = append(errs, err) + case <-timeout: + errs = append(errs, fmt.Errorf("Timeout while uploading to azure Blob")) + } + } + + if 0 < len(errs) { + return fmt.Errorf("Failed to upload json to Azure Blob: %v", errs) + } + return nil +} + +func (w AzureBlobWriter) upload(res models.ScanResult) (err error) { + cli, err := getBlobClient() + if err != nil { + return err + } + timestr := time.Now().Format("20060102_1504") + name := "" + if res.Container.ContainerID == "" { + name = fmt.Sprintf("%s/%s.json", timestr, res.ServerName) + } else { + name = fmt.Sprintf("%s/%s_%s.json", timestr, res.ServerName, res.Container.Name) + } + + jsonBytes, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("Failed to Marshal to JSON: %s", err) + } + + if err = cli.CreateBlockBlobFromReader( + c.Conf.AzureContainer, + name, + uint64(len(jsonBytes)), + bytes.NewReader(jsonBytes), + map[string]string{}, + ); err != nil { + return fmt.Errorf("%s/%s, %s", + c.Conf.AzureContainer, name, err) + } + return +} diff --git a/report/s3.go b/report/s3.go index 6e86d098..b0f87711 100644 --- a/report/s3.go +++ b/report/s3.go @@ -32,16 +32,6 @@ import ( "github.com/future-architect/vuls/models" ) -// S3Writer writes results to S3 -type S3Writer struct{} - -func getS3() *s3.S3 { - return s3.New(session.New(&aws.Config{ - Region: aws.String(c.Conf.AwsRegion), - Credentials: credentials.NewSharedCredentials("", c.Conf.AwsProfile), - })) -} - // CheckIfBucketExists check the existence of S3 bucket func CheckIfBucketExists() error { svc := getS3() @@ -67,7 +57,17 @@ func CheckIfBucketExists() error { return nil } -// Write put results in S3 +// S3Writer writes results to S3 +type S3Writer struct{} + +func getS3() *s3.S3 { + return s3.New(session.New(&aws.Config{ + Region: aws.String(c.Conf.AwsRegion), + Credentials: credentials.NewSharedCredentials("", c.Conf.AwsProfile), + })) +} + +// Write results to S3 func (w S3Writer) Write(scanResults []models.ScanResult) (err error) { var jsonBytes []byte diff --git a/scan/debian.go b/scan/debian.go index a1ac501c..9c675f92 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -508,6 +508,7 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac case err := <-errChan: errs = append(errs, err) case <-timeout: + //TODO append to errs return nil, fmt.Errorf("Timeout scanPackageCveIDs") } }