fix(report): fix cvedb-url, add -cvedb-type=http (#734)

* fix(report): fix cvedb-url, add -cvedb-type=http

* feat(report): support go-exploitdb server mode

* update deps

* implement tui

* fix server mode

* fix(tui): default value of cvedb-type to ""

* update deps

.dockerignore

@@ -0,0 +1,7 @@
+.dockerignore
+Dockerfile
+vendor/
+cve.sqlite3*
+oval.sqlite3*
+setup/
+img/

.github/ISSUE_TEMPLATE.md

@@ -1,7 +1,26 @@
 
-# Environment
+# What did you do? (required. The issue will be **closed** when not provided.)
 
-## Vuls
+
+# What did you expect to happen?
+
+
+# What happened instead?
+
+* Current Output
+
+Please re-run the command using ```-debug``` and provide the output below.
+
+# Steps to reproduce the behaviour
+
+
+# Configuration (**MUST** fill this out):
+
+* Go version (`go version`):
+
+* Go environment (`go env`):
+
+* Vuls environment:
 
 Hash : ____
 
@@ -9,28 +28,11 @@ To check the commit hash of HEAD
 $ vuls -v
 
 or
+
 $ cd $GOPATH/src/github.com/future-architect/vuls 
 $ git rev-parse --short HEAD 
 
-## OS
-- Target Server: Write here
-- Vuls Server: Write here
-
-## Go
-- Go version: here
-
-# Current Output
-
-Please re-run the command using ```-debug``` and provide the output below.
-
-# Addition Details
-
-Can you also please fill in each of the remaining sections.
-
-## Expected Behavior
-
-## Actual Behavior
-
-## Steps to reproduce the behaviour
+* config.toml:
 
+* command:
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,14 +1,26 @@
-## What did you implement:
 
-Closes #XXXXX
+If this Pull Request is work in progress, Add a prefix of “[WIP]” in the title.
 
-## How did you implement it:
+# What did you implement:
 
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. 
 
-## How can we verify it:
+Fixes # (issue)
 
+## Type of change
 
-## Todos:
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+# How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce.
+
+# Checklist:
 You don't have to satisfy all of the following.
 
 - [ ] Write tests
@@ -21,4 +33,8 @@ You don't have to satisfy all of the following.
 - [ ] Update the messages below
 
 ***Is this ready for review?:*** NO  
-***Is it a breaking change?:*** NO
+
+# Reference
+
+* https://blog.github.com/2015-01-21-how-to-write-the-perfect-pull-request/
+

.gitignore

@@ -1,6 +1,7 @@
 vuls
 .vscode
 *.txt
+*.json
 *.sqlite3*
 *.db
 tags

.travis.yml

@@ -1,7 +1,7 @@
 language: go
 
 go:
-  - 1.8
+  - "1.11.x"
 
 after_success:
   - test -n "$TRAVIS_TAG" && curl -sL https://git.io/goreleaser | bash

Dockerfile

@@ -0,0 +1,33 @@
+FROM golang:alpine as builder
+
+RUN apk add --no-cache \
+        git \
+        make \
+        gcc \
+        musl-dev
+
+ENV REPOSITORY github.com/future-architect/vuls
+COPY . $GOPATH/src/$REPOSITORY
+RUN cd $GOPATH/src/$REPOSITORY && make install
+
+
+FROM alpine:3.7
+
+MAINTAINER hikachan sadayuki-matsuno
+
+ENV LOGDIR /var/log/vuls
+ENV WORKDIR /vuls
+
+RUN apk add --no-cache \
+        openssh-client \
+        ca-certificates \
+    && mkdir -p $WORKDIR $LOGDIR
+
+COPY --from=builder /go/bin/vuls /usr/local/bin/
+
+VOLUME [$WORKDIR, $LOGDIR]
+WORKDIR $WORKDIR
+ENV PWD $WORKDIR
+
+ENTRYPOINT ["vuls"]
+CMD ["--help"]

GNUmakefile

@@ -15,21 +15,21 @@
 	clean
 
 SRCS = $(shell git ls-files '*.go')
-PKGS = ./. ./cache ./commands ./config ./models ./oval ./report ./scan ./util 
+PKGS = $(shell go list ./...)
 VERSION := $(shell git describe --tags --abbrev=0)
 REVISION := $(shell git rev-parse --short HEAD)
-LDFLAGS := -X 'main.version=$(VERSION)' \
-	-X 'main.revision=$(REVISION)'
+LDFLAGS := -X 'github.com/future-architect/vuls/config.Version=$(VERSION)' \
+	-X 'github.com/future-architect/vuls/config.Revision=$(REVISION)'
 
-all: dep build test
+all: dep build
 
 dep:
 	go get -u github.com/golang/dep/...
-	dep ensure
+	dep ensure -v
 
 depup:
 	go get -u github.com/golang/dep/...
-	dep ensure -update
+	dep ensure -update -v
 
 build: main.go dep pretest
 	go build -ldflags "$(LDFLAGS)" -o vuls $<
@@ -39,26 +39,28 @@ install: main.go dep pretest
 
 
 lint:
-	@ go get -v github.com/golang/lint/golint
-	$(foreach file,$(SRCS),golint $(file) || exit;)
+	@ go get -v golang.org/x/lint/golint
+	golint $(PKGS)
 
 vet:
 	#  @-go get -v golang.org/x/tools/cmd/vet
-	echo $(PKGS) | xargs go vet || exit;
+	go vet ./... || exit;
 
 fmt:
 	gofmt -s -w $(SRCS)
 
+mlint:
+	$(foreach file,$(SRCS),gometalinter $(file) || exit;)
+
 fmtcheck:
 	$(foreach file,$(SRCS),gofmt -s -d $(file);)
 
 pretest: lint vet fmtcheck
 
-test: pretest
-	go install
+test: 
 	echo $(PKGS) | xargs go test -cover -v || exit;
 
-unused :
+unused:
 	$(foreach pkg,$(PKGS),unused $(pkg);)
 
 cov:

Gopkg.toml

@@ -1,7 +1,6 @@
-
 # Gopkg.toml example
 #
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
 # for detailed Gopkg.toml documentation.
 #
 # required = ["github.com/user/thing/cmd/thing"]
@@ -17,78 +16,26 @@
 #   source = "github.com/myfork/project2"
 #
 # [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[constraint]]
-  name = "github.com/BurntSushi/toml"
-  version = "0.3.0"
-
-[[constraint]]
-  name = "github.com/asaskevich/govalidator"
-  version = "6.0.0"
-
-[[constraint]]
-  name = "github.com/boltdb/bolt"
-  version = "1.3.1"
-
-[[constraint]]
-  name = "github.com/cenkalti/backoff"
-  version = "1.0.0"
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
 
 [[constraint]]
+  name = "github.com/knqyf263/gost"
   branch = "master"
-  name = "github.com/google/subcommands"
 
 [[constraint]]
-  branch = "master"
-  name = "github.com/gosuri/uitable"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/howeyc/gopass"
-
-[[constraint]]
-  name = "github.com/jroimartin/gocui"
-  version = "0.3.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/k0kubun/pp"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/knqyf263/go-deb-version"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/knqyf263/go-rpm-version"
-
-[[constraint]]
-  name = "github.com/kotakanbe/go-pingscanner"
-  version = "0.1.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/kotakanbe/logrus-prefixed-formatter"
-
-[[constraint]]
-  name = "github.com/parnurzeal/gorequest"
-  version = "0.2.15"
-
-[[constraint]]
-  name = "github.com/rifflock/lfshook"
-  version = "1.7.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/sirupsen/logrus"
-
-[[constraint]]
-  branch = "master"
   name = "github.com/kotakanbe/go-cve-dictionary"
+  branch = "master"
 
 [[constraint]]
+  name = "github.com/mozqnet/go-exploitdb"
   branch = "master"
-  name = "github.com/kotakanbe/goval-dictionary"
+
+[prune]
+  go-tests = true
+  unused-packages = true

LICENSE

@@ -632,7 +632,7 @@ state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
     Vuls - Vulnerability Scanner
-    Copyright (C) 2016  Future Architect, Inc. Japan.
+    Copyright (C) 2016  Future Corporation , Japan.
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -652,7 +652,7 @@ Also add information on how to contact you by electronic and paper mail.
   If the program does terminal interaction, make it output a short
 notice like this when it starts in an interactive mode:
 
-    Vuls  Copyright (C) 2016  Future Architect, Inc. Japan.
+    Vuls  Copyright (C) 2016  Future Corporation , Japan.
     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
     This is free software, and you are welcome to redistribute it
     under certain conditions; type `show c' for details.

NOTICE

@@ -1,2 +1,2 @@
-Vuls Copyright (C) 2016  Future Architect, Inc. Japan.
+Vuls Copyright (C) 2016  Future Corporation , Japan.
 

cache/bolt.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -183,9 +183,6 @@ func (b Bolt) PutChangelog(servername, packName, changelog string) error {
 		if bkt == nil {
 			return fmt.Errorf("Failed to get Bucket: %s", servername)
 		}
-		if err := bkt.Put([]byte(packName), []byte(changelog)); err != nil {
-			return err
-		}
-		return nil
+		return bkt.Put([]byte(packName), []byte(changelog))
 	})
 }

cache/bolt_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

cache/db.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

commands/configtest.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -33,15 +33,8 @@ import (
 // ConfigtestCmd is Subcommand
 type ConfigtestCmd struct {
 	configPath     string
-	logDir         string
 	askKeyPassword bool
-	containersOnly bool
-	deep           bool
-	sshNative      bool
-	httpProxy      string
 	timeoutSec     int
-
-	debug bool
 }
 
 // Name return subcommand name
@@ -54,7 +47,6 @@ func (*ConfigtestCmd) Synopsis() string { return "Test configuration" }
 func (*ConfigtestCmd) Usage() string {
 	return `configtest:
 	configtest
-			[-deep]
 			[-config=/path/to/config.toml]
 			[-log-dir=/path/to/log]
 			[-ask-key-password]
@@ -63,6 +55,7 @@ func (*ConfigtestCmd) Usage() string {
 			[-containers-only]
 			[-http-proxy=http://192.168.0.1:8080]
 			[-debug]
+			[-vvv]
 
 			[SERVER]...
 `
@@ -75,48 +68,40 @@ func (p *ConfigtestCmd) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml")
 
 	defaultLogDir := util.GetDefaultLogDir()
-	f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log")
-
-	f.BoolVar(&p.debug, "debug", false, "debug mode")
+	f.StringVar(&c.Conf.LogDir, "log-dir", defaultLogDir, "/path/to/log")
+	f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode")
 
 	f.IntVar(&p.timeoutSec, "timeout", 5*60, "Timeout(Sec)")
 
-	f.BoolVar(
-		&p.askKeyPassword,
-		"ask-key-password",
-		false,
+	f.BoolVar(&p.askKeyPassword, "ask-key-password", false,
 		"Ask ssh privatekey password before scanning",
 	)
 
-	f.BoolVar(&p.deep, "deep", false, "Config test for deep scan mode")
+	f.StringVar(&c.Conf.HTTPProxy, "http-proxy", "",
+		"http://proxy-url:port (default: empty)")
 
-	f.StringVar(
-		&p.httpProxy,
-		"http-proxy",
-		"",
-		"http://proxy-url:port (default: empty)",
-	)
-
-	f.BoolVar(
-		&p.sshNative,
-		"ssh-native-insecure",
-		false,
+	f.BoolVar(&c.Conf.SSHNative, "ssh-native-insecure", false,
 		"Use Native Go implementation of SSH. Default: Use the external command")
 
-	f.BoolVar(
-		&p.containersOnly,
-		"containers-only",
-		false,
+	f.BoolVar(&c.Conf.SSHConfig, "ssh-config", false,
+		"Use SSH options specified in ssh_config preferentially")
+
+	f.BoolVar(&c.Conf.ContainersOnly, "containers-only", false,
 		"Test containers only. Default: Test both of hosts and containers")
+
+	f.BoolVar(&c.Conf.Vvv, "vvv", false, "ssh -vvv")
 }
 
 // Execute execute
 func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
 	// Setup Logger
-	c.Conf.Debug = p.debug
-	c.Conf.LogDir = p.logDir
 	util.Log = util.NewCustomLogger(c.ServerInfo{})
 
+	if err := mkdirDotVuls(); err != nil {
+		util.Log.Errorf("Failed to create .vuls: %s", err)
+		return subcommands.ExitUsageError
+	}
+
 	var keyPass string
 	var err error
 	if p.askKeyPassword {
@@ -134,10 +119,6 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa
 		util.Log.Errorf("Please check README: https://github.com/future-architect/vuls#configuration")
 		return subcommands.ExitUsageError
 	}
-	c.Conf.SSHNative = p.sshNative
-	c.Conf.HTTPProxy = p.httpProxy
-	c.Conf.ContainersOnly = p.containersOnly
-	c.Conf.Deep = p.deep
 
 	var servernames []string
 	if 0 < len(f.Args()) {
@@ -174,12 +155,22 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa
 		return subcommands.ExitFailure
 	}
 
+	util.Log.Info("Checking Scan Modes...")
+	if err := scan.CheckScanModes(); err != nil {
+		util.Log.Errorf("Fix config.toml: %s", err)
+		return subcommands.ExitFailure
+	}
+
 	util.Log.Info("Checking dependencies...")
 	scan.CheckDependencies(p.timeoutSec)
 
 	util.Log.Info("Checking sudo settings...")
 	scan.CheckIfSudoNoPasswd(p.timeoutSec)
 
-	scan.PrintSSHableServerNames()
-	return subcommands.ExitSuccess
+	util.Log.Info("It can be scanned with fast scan mode even if warn or err messages are displayed due to lack of dependent packages or sudo settings in fast-root or deep scan mode")
+
+	if scan.PrintSSHableServerNames() {
+		return subcommands.ExitSuccess
+	}
+	return subcommands.ExitFailure
 }

commands/discover.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -90,61 +90,133 @@ func (p *DiscoverCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface
 // Output the template of config.toml
 func printConfigToml(ips []string) (err error) {
 	const tomlTemplate = `
-[slack]
-hookURL      = "https://hooks.slack.com/services/abc123/defghijklmnopqrstuvwxyz"
-#legacyToken  = "xoxp-11111111111-222222222222-3333333333"
-channel      = "#channel-name"
-#channel      = "${servername}"
-iconEmoji    = ":ghost:"
-authUser     = "username"
-notifyUsers  = ["@username"]
 
-[email]
-smtpAddr      = "smtp.example.com"
-smtpPort      = "587"
-user          = "username"
-password      = "password"
-from          = "from@example.com"
-to            = ["to@example.com"]
-cc            = ["cc@example.com"]
-subjectPrefix = "[vuls]"
+# https://vuls.io/docs/en/usage-settings.html
+[cveDict]
+type        = "sqlite3"
+sqlite3Path = "/path/to/cve.sqlite3"
+#url        = ""
 
+[ovalDict]
+type        = "sqlite3"
+sqlite3Path = "/path/to/oval.sqlite3"
+#url        = ""
+
+[gost]
+type        = "sqlite3"
+sqlite3Path = "/path/to/gost.sqlite3"
+#url        = ""
+
+[exploit]
+type        = "sqlite3"
+sqlite3Path = "/path/to/go-exploitdb.sqlite3"
+#url        = ""
+
+# https://vuls.io/docs/en/usage-settings.html#slack-section
+#[slack]
+#hookURL      = "https://hooks.slack.com/services/abc123/defghijklmnopqrstuvwxyz"
+##legacyToken = "xoxp-11111111111-222222222222-3333333333"
+#channel      = "#channel-name"
+##channel     = "${servername}"
+#iconEmoji    = ":ghost:"
+#authUser     = "username"
+#notifyUsers  = ["@username"]
+
+# https://vuls.io/docs/en/usage-settings.html#email-section
+#[email]
+#smtpAddr      = "smtp.example.com"
+#smtpPort      = "587"
+#user          = "username"
+#password      = "password"
+#from          = "from@example.com"
+#to            = ["to@example.com"]
+#cc            = ["cc@example.com"]
+#subjectPrefix = "[vuls]"
+
+# https://vuls.io/docs/en/usage-settings.html#http-section
+#[http]
+#url = "http://localhost:11234"
+
+# https://vuls.io/docs/en/usage-settings.html#syslog-section
+#[syslog]
+#protocol    = "tcp"
+#host        = "localhost"
+#port        = "514"
+#tag         = "vuls"
+#facility    = "local0"
+#severity    = "alert"
+#verbose     = false
+
+# https://vuls.io/docs/en/usage-report.html#example-put-results-in-s3-bucket
+#[aws]
+#profile                = "default"
+#region                 = "ap-northeast-1"
+#s3Bucket               = "vuls"
+#s3ResultsDir           = "/path/to/result"
+#s3ServerSideEncryption = "AES256"
+
+# https://vuls.io/docs/en/usage-report.html#example-put-results-in-azure-blob-storage<Paste>
+#[azure]
+#accountName   = "default"
+#accountKey    = "xxxxxxxxxxxxxx"
+#containerName = "vuls"
+
+# https://vuls.io/docs/en/usage-settings.html#stride-section
+#[stride]
+#hookURL   = "xxxxxxxxxxxxxxx"
+#authToken = "xxxxxxxxxxxxxx"
+
+# https://vuls.io/docs/en/usage-settings.html#hipchat-section
+#[hipchat]
+#room      = "vuls"
+#authToken = "xxxxxxxxxxxxxx"
+
+# https://vuls.io/docs/en/usage-settings.html#chatwork-section
+#[chatwork]
+#room     = "xxxxxxxxxxx"
+#apiToken = "xxxxxxxxxxxxxxxxxx"
+
+# https://vuls.io/docs/en/usage-settings.html#default-section
 [default]
-#port        = "22"
-#user        = "username"
-#keyPath     = "/home/username/.ssh/id_rsa"
+#port               = "22"
+#user               = "username"
+#keyPath            = "/home/username/.ssh/id_rsa"
+#scanMode           = ["fast", "fast-root", "deep", "offline"]
 #cpeNames = [
 #  "cpe:/a:rubyonrails:ruby_on_rails:4.2.1",
 #]
-#dependencyCheckXMLPath = "/tmp/dependency-check-report.xml"
-#ignoreCves = ["CVE-2014-6271"]
-#optional = [
-#    ["key", "value"],
-#]
-#containers = ["${running}"]
-
+#owaspDCXMLPath     = "/tmp/dependency-check-report.xml"
+#ignoreCves         = ["CVE-2014-6271"]
+#containerType      = "docker" #or "lxd" or "lxc" default: docker
+#containersIncluded = ["${running}"]
+#containersExcluded = ["container_name_a"]
 
+# https://vuls.io/docs/en/usage-settings.html#servers-section
 [servers]
 {{- $names:=  .Names}}
 {{range $i, $ip := .IPs}}
 [servers.{{index $names $i}}]
-host         = "{{$ip}}"
-#port        = "22"
-#user        = "root"
-#keyPath     = "/home/username/.ssh/id_rsa"
-#type 		 = "pseudo"
-#cpeNames = [
-#  "cpe:/a:rubyonrails:ruby_on_rails:4.2.1",
-#]
-#dependencyCheckXMLPath = "/tmp/dependency-check-report.xml"
-#ignoreCves = ["CVE-2014-0160"]
-#optional = [
-#    ["key", "value"],
-#]
-#[servers.{{index $names $i}}.containers]
-#type = "docker" #or "lxd" default: docker
-#includes = ["${running}"]
-#excludes = ["container_name_a", "4aa37a8b63b9"]
+host                = "{{$ip}}"
+#port               = "22"
+#user               = "root"
+#keyPath            = "/home/username/.ssh/id_rsa"
+#scanMode           = ["fast", "fast-root", "deep", "offline"]
+#type               = "pseudo"
+#memo               = "DB Server"
+#cpeNames           = [ "cpe:/a:rubyonrails:ruby_on_rails:4.2.1" ]
+#owaspDCXMLPath     = "/path/to/dependency-check-report.xml"
+#ignoreCves         = ["CVE-2014-0160"]
+#containerType      = "docker" #or "lxd" or "lxc" default: docker
+#containersIncluded = ["${running}"]
+#containersExcluded = ["container_name_a"]
+
+#[servers.{{index $names $i}}.containers.container_name_a]
+#cpeNames       = [ "cpe:/a:rubyonrails:ruby_on_rails:4.2.1" ]
+#owaspDCXMLPath = "/path/to/dependency-check-report.xml"
+#ignoreCves     = ["CVE-2014-0160"]
+
+#[servers.{{index $names $i}}.optional]
+#key = "value1"
 
 
 {{end}}

commands/history.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -32,11 +32,7 @@ import (
 )
 
 // HistoryCmd is Subcommand of list scanned results
-type HistoryCmd struct {
-	debug      bool
-	debugSQL   bool
-	resultsDir string
-}
+type HistoryCmd struct{}
 
 // Name return subcommand name
 func (*HistoryCmd) Name() string { return "history" }
@@ -56,19 +52,16 @@ func (*HistoryCmd) Usage() string {
 
 // SetFlags set flag
 func (p *HistoryCmd) SetFlags(f *flag.FlagSet) {
-	f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode")
+	f.BoolVar(&c.Conf.DebugSQL, "debug-sql", false, "SQL debug mode")
 
 	wd, _ := os.Getwd()
 	defaultResultsDir := filepath.Join(wd, "results")
-	f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results")
+	f.StringVar(&c.Conf.ResultsDir, "results-dir", defaultResultsDir, "/path/to/results")
 }
 
 // Execute execute
 func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
 
-	c.Conf.DebugSQL = p.debugSQL
-	c.Conf.ResultsDir = p.resultsDir
-
 	dirs, err := report.ListValidJSONDirs()
 	if err != nil {
 		return subcommands.ExitFailure

commands/report.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,69 +20,29 @@ package commands
 import (
 	"context"
 	"flag"
-	"fmt"
 	"os"
 	"path/filepath"
 
 	c "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/exploit"
+	"github.com/future-architect/vuls/gost"
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/oval"
 	"github.com/future-architect/vuls/report"
 	"github.com/future-architect/vuls/util"
 	"github.com/google/subcommands"
+	"github.com/k0kubun/pp"
+	cvelog "github.com/kotakanbe/go-cve-dictionary/log"
 )
 
 // ReportCmd is subcommand for reporting
 type ReportCmd struct {
-	lang       string
-	debug      bool
-	debugSQL   bool
-	configPath string
-	resultsDir string
-	logDir     string
-	refreshCve bool
-
-	cvssScoreOver      float64
-	ignoreUnscoredCves bool
-	ignoreUnfixed      bool
-
-	httpProxy string
-
-	cveDBType string
-	cveDBPath string
-	cveDBURL  string
-
-	ovalDBType string
-	ovalDBPath string
-	ovalDBURL  string
-
-	toSlack     bool
-	toEMail     bool
-	toLocalFile bool
-	toS3        bool
-	toAzureBlob bool
-
-	formatJSON        bool
-	formatXML         bool
-	formatOneEMail    bool
-	formatOneLineText bool
-	formatShortText   bool
-	formatFullText    bool
-
-	gzip bool
-
-	awsProfile      string
-	awsS3Bucket     string
-	awsS3ResultsDir string
-	awsRegion       string
-
-	azureAccount   string
-	azureKey       string
-	azureContainer string
-
-	pipe bool
-
-	diff bool
+	configPath  string
+	cveDict     c.GoCveDictConf
+	ovalDict    c.GovalDictConf
+	gostConf    c.GostConf
+	exploitConf c.ExploitConf
+	httpConf    c.HTTPConf
 }
 
 // Name return subcommand name
@@ -100,39 +60,45 @@ func (*ReportCmd) Usage() string {
 		[-results-dir=/path/to/results]
 		[-log-dir=/path/to/log]
 		[-refresh-cve]
-		[-cvedb-type=sqlite3|mysql|postgres]
-		[-cvedb-path=/path/to/cve.sqlite3]
-		[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
-		[-ovaldb-type=sqlite3|mysql]
-		[-ovaldb-path=/path/to/oval.sqlite3]
-		[-ovaldb-url=http://127.0.0.1:1324 or DB connection string]
 		[-cvss-over=7]
 		[-diff]
 		[-ignore-unscored-cves]
 		[-ignore-unfixed]
 		[-to-email]
+		[-to-http]
 		[-to-slack]
+		[-to-stride]
+		[-to-hipchat]
+		[-to-chatwork]
 		[-to-localfile]
 		[-to-s3]
 		[-to-azure-blob]
+		[-to-saas]
 		[-format-json]
 		[-format-xml]
 		[-format-one-email]
 		[-format-one-line-text]
-		[-format-short-text]
+		[-format-list]
 		[-format-full-text]
 		[-gzip]
-		[-aws-profile=default]
-		[-aws-region=us-west-2]
-		[-aws-s3-bucket=bucket_name]
-		[-aws-s3-results-dir=/bucket/path/to/results]
-		[-azure-account=account]
-		[-azure-key=key]
-		[-azure-container=container]
+		[-uuid]
 		[-http-proxy=http://192.168.0.1:8080]
 		[-debug]
 		[-debug-sql]
 		[-pipe]
+		[-cvedb-type=sqlite3|mysql|postgres|redis|http]
+		[-cvedb-sqlite3-path=/path/to/cve.sqlite3]
+		[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
+		[-ovaldb-type=sqlite3|mysql|redis|http]
+		[-ovaldb-sqlite3-path=/path/to/oval.sqlite3]
+		[-ovaldb-url=http://127.0.0.1:1324 or DB connection string]
+		[-gostdb-type=sqlite3|mysql|redis|http]
+		[-gostdb-sqlite3-path=/path/to/gost.sqlite3]
+		[-gostdb-url=http://127.0.0.1:1325 or DB connection string]
+		[-exploitdb-type=sqlite3|mysql|redis|http]
+		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
+		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
+		[-http="http://vuls-report-server"]
 
 		[RFC3339 datetime format under results dir]
 `
@@ -140,204 +106,117 @@ func (*ReportCmd) Usage() string {
 
 // SetFlags set flag
 func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
-	f.StringVar(&p.lang, "lang", "en", "[en|ja]")
-	f.BoolVar(&p.debug, "debug", false, "debug mode")
-	f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode")
+	f.StringVar(&c.Conf.Lang, "lang", "en", "[en|ja]")
+	f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode")
+	f.BoolVar(&c.Conf.DebugSQL, "debug-sql", false, "SQL debug mode")
 
 	wd, _ := os.Getwd()
-
 	defaultConfPath := filepath.Join(wd, "config.toml")
 	f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml")
 
 	defaultResultsDir := filepath.Join(wd, "results")
-	f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results")
+	f.StringVar(&c.Conf.ResultsDir, "results-dir", defaultResultsDir, "/path/to/results")
 
 	defaultLogDir := util.GetDefaultLogDir()
-	f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log")
+	f.StringVar(&c.Conf.LogDir, "log-dir", defaultLogDir, "/path/to/log")
 
-	f.BoolVar(
-		&p.refreshCve,
-		"refresh-cve",
-		false,
+	f.BoolVar(&c.Conf.RefreshCve, "refresh-cve", false,
 		"Refresh CVE information in JSON file under results dir")
 
-	f.StringVar(
-		&p.cveDBType,
-		"cvedb-type",
-		"sqlite3",
-		"DB type for fetching CVE dictionary (sqlite3, mysql or postgres)")
-
-	defaultCveDBPath := filepath.Join(wd, "cve.sqlite3")
-	f.StringVar(
-		&p.cveDBPath,
-		"cvedb-path",
-		defaultCveDBPath,
-		"/path/to/sqlite3 (For get cve detail from cve.sqlite3)")
-
-	f.StringVar(
-		&p.cveDBURL,
-		"cvedb-url",
-		"",
-		"http://cve-dictionary.com:1323 or mysql connection string")
-
-	f.StringVar(
-		&p.ovalDBType,
-		"ovaldb-type",
-		"sqlite3",
-		"DB type for fetching OVAL dictionary (sqlite3 or mysql)")
-
-	defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3")
-	f.StringVar(
-		&p.ovalDBPath,
-		"ovaldb-path",
-		defaultOvalDBPath,
-		"/path/to/sqlite3 (For get oval detail from oval.sqlite3)")
-
-	f.StringVar(
-		&p.ovalDBURL,
-		"ovaldb-url",
-		"",
-		"http://goval-dictionary.com:1324 or mysql connection string")
-
-	f.Float64Var(
-		&p.cvssScoreOver,
-		"cvss-over",
-		0,
+	f.Float64Var(&c.Conf.CvssScoreOver, "cvss-over", 0,
 		"-cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))")
 
-	f.BoolVar(&p.diff,
-		"diff",
-		false,
-		fmt.Sprintf("Difference between previous result and current result "))
+	f.BoolVar(&c.Conf.Diff, "diff", false,
+		"Difference between previous result and current result ")
 
-	f.BoolVar(
-		&p.ignoreUnscoredCves,
-		"ignore-unscored-cves",
-		false,
+	f.BoolVar(&c.Conf.IgnoreUnscoredCves, "ignore-unscored-cves", false,
 		"Don't report the unscored CVEs")
 
 	f.BoolVar(
-		&p.ignoreUnfixed,
-		"ignore-unfixed",
-		false,
+		&c.Conf.IgnoreUnfixed, "ignore-unfixed", false,
 		"Don't report the unfixed CVEs")
 
 	f.StringVar(
-		&p.httpProxy,
-		"http-proxy",
-		"",
+		&c.Conf.HTTPProxy, "http-proxy", "",
 		"http://proxy-url:port (default: empty)")
 
-	f.BoolVar(&p.formatJSON,
-		"format-json",
-		false,
-		fmt.Sprintf("JSON format"))
-
-	f.BoolVar(&p.formatXML,
-		"format-xml",
-		false,
-		fmt.Sprintf("XML format"))
-
-	f.BoolVar(&p.formatOneEMail,
-		"format-one-email",
-		false,
+	f.BoolVar(&c.Conf.FormatJSON, "format-json", false, "JSON format")
+	f.BoolVar(&c.Conf.FormatXML, "format-xml", false, "XML format")
+	f.BoolVar(&c.Conf.FormatOneEMail, "format-one-email", false,
 		"Send all the host report via only one EMail (Specify with -to-email)")
+	f.BoolVar(&c.Conf.FormatOneLineText, "format-one-line-text", false,
+		"One line summary in plain text")
+	f.BoolVar(&c.Conf.FormatList, "format-list", false, "Display as list format")
+	f.BoolVar(&c.Conf.FormatFullText, "format-full-text", false,
+		"Detail report in plain text")
 
-	f.BoolVar(&p.formatOneLineText,
-		"format-one-line-text",
-		false,
-		fmt.Sprintf("One line summary in plain text"))
-
-	f.BoolVar(&p.formatShortText,
-		"format-short-text",
-		false,
-		fmt.Sprintf("Summary in plain text"))
-
-	f.BoolVar(&p.formatFullText,
-		"format-full-text",
-		false,
-		fmt.Sprintf("Detail report in plain text"))
-
-	f.BoolVar(&p.gzip, "gzip", false, "gzip compression")
-
-	f.BoolVar(&p.toSlack, "to-slack", false, "Send report via Slack")
-	f.BoolVar(&p.toEMail, "to-email", false, "Send report via Email")
-	f.BoolVar(&p.toLocalFile,
-		"to-localfile",
-		false,
-		fmt.Sprintf("Write report to localfile"))
-
-	f.BoolVar(&p.toS3,
-		"to-s3",
-		false,
+	f.BoolVar(&c.Conf.ToSlack, "to-slack", false, "Send report via Slack")
+	f.BoolVar(&c.Conf.ToStride, "to-stride", false, "Send report via Stride")
+	f.BoolVar(&c.Conf.ToHipChat, "to-hipchat", false, "Send report via hipchat")
+	f.BoolVar(&c.Conf.ToChatWork, "to-chatwork", false, "Send report via chatwork")
+	f.BoolVar(&c.Conf.ToEmail, "to-email", false, "Send report via Email")
+	f.BoolVar(&c.Conf.ToSyslog, "to-syslog", false, "Send report via Syslog")
+	f.BoolVar(&c.Conf.ToLocalFile, "to-localfile", false, "Write report to localfile")
+	f.BoolVar(&c.Conf.ToS3, "to-s3", false,
 		"Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json/xml/txt)")
-	f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS profile to use")
-	f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS region to use")
-	f.StringVar(&p.awsS3Bucket, "aws-s3-bucket", "", "S3 bucket name")
-	f.StringVar(&p.awsS3ResultsDir, "aws-s3-results-dir", "", "/bucket/path/to/results")
-
-	f.BoolVar(&p.toAzureBlob,
-		"to-azure-blob",
-		false,
+	f.BoolVar(&c.Conf.ToHTTP, "to-http", false, "Send report via HTTP POST")
+	f.BoolVar(&c.Conf.ToAzureBlob, "to-azure-blob", false,
 		"Write report to Azure Storage blob (container/yyyyMMdd_HHmm/servername.json/xml/txt)")
-	f.StringVar(&p.azureAccount,
-		"azure-account",
-		"",
-		"Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified")
-	f.StringVar(&p.azureKey,
-		"azure-key",
-		"",
-		"Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified")
-	f.StringVar(&p.azureContainer, "azure-container", "", "Azure storage container name")
+	f.BoolVar(&c.Conf.ToSaas, "to-saas", false,
+		"Upload report to Future Vuls(https://vuls.biz/) before report")
+
+	f.BoolVar(&c.Conf.GZIP, "gzip", false, "gzip compression")
+	f.BoolVar(&c.Conf.UUID, "uuid", false,
+		"Auto generate of scan target servers and then write to config.toml and scan result")
+	f.BoolVar(&c.Conf.Pipe, "pipe", false, "Use args passed via PIPE")
+
+	f.StringVar(&p.cveDict.Type, "cvedb-type", "",
+		"DB type of go-cve-dictionary (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.cveDict.SQLite3Path, "cvedb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.cveDict.URL, "cvedb-url", "",
+		"http://go-cve-dictionary.com:1323 or DB connection string")
+
+	f.StringVar(&p.ovalDict.Type, "ovaldb-type", "",
+		"DB type of goval-dictionary (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.ovalDict.SQLite3Path, "ovaldb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.ovalDict.URL, "ovaldb-url", "",
+		"http://goval-dictionary.com:1324 or DB connection string")
+
+	f.StringVar(&p.gostConf.Type, "gostdb-type", "",
+		"DB type of gost (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.gostConf.SQLite3Path, "gostdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.gostConf.URL, "gostdb-url", "",
+		"http://gost.com:1325 or DB connection string")
+
+	f.StringVar(&p.exploitConf.Type, "exploitdb-type", "",
+		"DB type of exploit (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.exploitConf.SQLite3Path, "exploitdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.exploitConf.URL, "exploitdb-url", "",
+		"http://exploit.com:1326 or DB connection string")
+
+	f.StringVar(&p.httpConf.URL, "http", "", "-to-http http://vuls-report")
 
-	f.BoolVar(
-		&p.pipe,
-		"pipe",
-		false,
-		"Use args passed via PIPE")
 }
 
 // Execute execute
 func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
-	c.Conf.Debug = p.debug
-	c.Conf.DebugSQL = p.debugSQL
-	c.Conf.LogDir = p.logDir
 	util.Log = util.NewCustomLogger(c.ServerInfo{})
+	cvelog.SetLogger(c.Conf.LogDir, false, c.Conf.Debug, false)
 
 	if err := c.Load(p.configPath, ""); err != nil {
 		util.Log.Errorf("Error loading %s, %s", p.configPath, err)
 		return subcommands.ExitUsageError
 	}
 
-	c.Conf.Lang = p.lang
-	c.Conf.ResultsDir = p.resultsDir
-	c.Conf.RefreshCve = p.refreshCve
-	c.Conf.Diff = p.diff
-	c.Conf.CveDBType = p.cveDBType
-	c.Conf.CveDBPath = p.cveDBPath
-	c.Conf.CveDBURL = p.cveDBURL
-	c.Conf.OvalDBType = p.ovalDBType
-	c.Conf.OvalDBPath = p.ovalDBPath
-	c.Conf.OvalDBURL = p.ovalDBURL
-	c.Conf.CvssScoreOver = p.cvssScoreOver
-	c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves
-	c.Conf.IgnoreUnfixed = p.ignoreUnfixed
-	c.Conf.HTTPProxy = p.httpProxy
-
-	c.Conf.FormatXML = p.formatXML
-	c.Conf.FormatJSON = p.formatJSON
-	c.Conf.FormatOneEMail = p.formatOneEMail
-	c.Conf.FormatOneLineText = p.formatOneLineText
-	c.Conf.FormatShortText = p.formatShortText
-	c.Conf.FormatFullText = p.formatFullText
-
-	c.Conf.GZIP = p.gzip
-	c.Conf.Diff = p.diff
-	c.Conf.Pipe = p.pipe
+	c.Conf.CveDict.Overwrite(p.cveDict)
+	c.Conf.OvalDict.Overwrite(p.ovalDict)
+	c.Conf.Gost.Overwrite(p.gostConf)
+	c.Conf.Exploit.Overwrite(p.exploitConf)
+	c.Conf.HTTP.Overwrite(p.httpConf)
 
 	var dir string
 	var err error
-	if p.diff {
+	if c.Conf.Diff {
 		dir, err = report.JSONDir([]string{})
 	} else {
 		dir, err = report.JSONDir(f.Args())
@@ -352,96 +231,181 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 		report.StdoutWriter{},
 	}
 
-	if p.toSlack {
+	if c.Conf.ToSlack {
 		reports = append(reports, report.SlackWriter{})
 	}
 
-	if p.toEMail {
+	if c.Conf.ToStride {
+		reports = append(reports, report.StrideWriter{})
+	}
+
+	if c.Conf.ToHipChat {
+		reports = append(reports, report.HipChatWriter{})
+	}
+
+	if c.Conf.ToChatWork {
+		reports = append(reports, report.ChatWorkWriter{})
+	}
+
+	if c.Conf.ToEmail {
 		reports = append(reports, report.EMailWriter{})
 	}
 
-	if p.toLocalFile {
+	if c.Conf.ToSyslog {
+		reports = append(reports, report.SyslogWriter{})
+	}
+
+	if c.Conf.ToHTTP {
+		reports = append(reports, report.HTTPRequestWriter{})
+	}
+
+	if c.Conf.ToLocalFile {
 		reports = append(reports, report.LocalFileWriter{
 			CurrentDir: dir,
 		})
 	}
 
-	if p.toS3 {
-		c.Conf.AwsRegion = p.awsRegion
-		c.Conf.AwsProfile = p.awsProfile
-		c.Conf.S3Bucket = p.awsS3Bucket
-		c.Conf.S3ResultsDir = p.awsS3ResultsDir
+	if c.Conf.ToS3 {
 		if err := report.CheckIfBucketExists(); err != nil {
-			util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %s", c.Conf.S3Bucket, err)
+			util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %s",
+				c.Conf.AWS.S3Bucket, err)
 			return subcommands.ExitUsageError
 		}
 		reports = append(reports, report.S3Writer{})
 	}
 
-	if p.toAzureBlob {
-		c.Conf.AzureAccount = p.azureAccount
-		if len(c.Conf.AzureAccount) == 0 {
-			c.Conf.AzureAccount = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if c.Conf.ToAzureBlob {
+		if len(c.Conf.Azure.AccountName) == 0 {
+			c.Conf.Azure.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
 		}
 
-		c.Conf.AzureKey = p.azureKey
-		if len(c.Conf.AzureKey) == 0 {
-			c.Conf.AzureKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+		if len(c.Conf.Azure.AccountKey) == 0 {
+			c.Conf.Azure.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
 		}
 
-		c.Conf.AzureContainer = p.azureContainer
-		if len(c.Conf.AzureContainer) == 0 {
+		if len(c.Conf.Azure.ContainerName) == 0 {
 			util.Log.Error("Azure storage container name is required with -azure-container option")
 			return subcommands.ExitUsageError
 		}
 		if err := report.CheckIfAzureContainerExists(); err != nil {
-			util.Log.Errorf("Check if there is a container beforehand: %s, err: %s", c.Conf.AzureContainer, err)
+			util.Log.Errorf("Check if there is a container beforehand: %s, err: %s",
+				c.Conf.Azure.ContainerName, err)
 			return subcommands.ExitUsageError
 		}
 		reports = append(reports, report.AzureBlobWriter{})
 	}
 
-	if !(p.formatJSON || p.formatOneLineText ||
-		p.formatShortText || p.formatFullText || p.formatXML) {
-		c.Conf.FormatShortText = true
+	if c.Conf.ToSaas {
+		if !c.Conf.UUID {
+			util.Log.Errorf("If you use the -to-saas option, you need to enable the uuid option")
+			return subcommands.ExitUsageError
+		}
+		reports = append(reports, report.SaasWriter{})
+	}
+
+	if !(c.Conf.FormatJSON || c.Conf.FormatOneLineText ||
+		c.Conf.FormatList || c.Conf.FormatFullText || c.Conf.FormatXML) {
+		c.Conf.FormatList = true
 	}
 
 	util.Log.Info("Validating config...")
 	if !c.Conf.ValidateOnReport() {
 		return subcommands.ExitUsageError
 	}
-	if err := report.CveClient.CheckHealth(); err != nil {
-		util.Log.Errorf("CVE HTTP server is not running. err: %s", err)
-		util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with -cvedb-path option")
-		return subcommands.ExitFailure
-	}
-	if c.Conf.CveDBURL != "" {
-		util.Log.Infof("cve-dictionary: %s", c.Conf.CveDBURL)
-	} else {
-		if c.Conf.CveDBType == "sqlite3" {
-			util.Log.Infof("cve-dictionary: %s", c.Conf.CveDBPath)
-		}
-	}
 
-	if c.Conf.OvalDBURL != "" {
-		err := oval.Base{}.CheckHTTPHealth()
-		if err != nil {
-			util.Log.Errorf("OVAL HTTP server is not running. err: %s", err)
-			util.Log.Errorf("Run goval-dictionary as server mode before reporting or run with -ovaldb-path option")
-			return subcommands.ExitFailure
-		}
-	}
-
-	var res models.ScanResults
-	if res, err = report.LoadScanResults(dir); err != nil {
+	var loaded models.ScanResults
+	if loaded, err = report.LoadScanResults(dir); err != nil {
 		util.Log.Error(err)
 		return subcommands.ExitFailure
 	}
 	util.Log.Infof("Loaded: %s", dir)
 
-	if res, err = report.FillCveInfos(res, dir); err != nil {
-		util.Log.Error(err)
-		return subcommands.ExitFailure
+	var res models.ScanResults
+	for _, r := range loaded {
+		if len(r.Errors) == 0 {
+			res = append(res, r)
+		} else {
+			util.Log.Warnf("Ignored since errors occurred during scanning: %s",
+				r.ServerName)
+		}
+	}
+
+	for _, r := range res {
+		util.Log.Debugf("%s: %s",
+			r.ServerInfo(),
+			pp.Sprintf("%s", c.Conf.Servers[r.ServerName]))
+	}
+
+	if c.Conf.UUID {
+		// Ensure UUIDs of scan target servers in config.toml
+		if err := report.EnsureUUIDs(p.configPath, res); err != nil {
+			util.Log.Errorf("Failed to ensure UUIDs: %s", err)
+			return subcommands.ExitFailure
+		}
+	}
+
+	if !c.Conf.ToSaas {
+		util.Log.Info("Validating db config...")
+		if !c.Conf.ValidateOnReportDB() {
+			return subcommands.ExitUsageError
+		}
+
+		if c.Conf.CveDict.URL != "" {
+			if err := report.CveClient.CheckHealth(); err != nil {
+				util.Log.Errorf("CVE HTTP server is not running. err: %s", err)
+				util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with `-cvedb-type=sqlite3 -cvedb-sqlite3-path` option instead of -cvedb-url")
+				return subcommands.ExitFailure
+			}
+		}
+
+		if c.Conf.OvalDict.URL != "" {
+			err := oval.Base{}.CheckHTTPHealth()
+			if err != nil {
+				util.Log.Errorf("OVAL HTTP server is not running. err: %s", err)
+				util.Log.Errorf("Run goval-dictionary as server mode before reporting or run with `-ovaldb-type=sqlite3 -ovaldb-sqlite3-path` option instead of -ovaldb-url")
+				return subcommands.ExitFailure
+			}
+		}
+
+		if c.Conf.Gost.URL != "" {
+			util.Log.Infof("gost: %s", c.Conf.Gost.URL)
+			err := gost.Base{}.CheckHTTPHealth()
+			if err != nil {
+				util.Log.Errorf("gost HTTP server is not running. err: %s", err)
+				util.Log.Errorf("Run gost as server mode before reporting or run with `-gostdb-type=sqlite3 -gostdb-sqlite3-path` option instead of -gostdb-url")
+				return subcommands.ExitFailure
+			}
+		}
+
+		if c.Conf.Exploit.URL != "" {
+			err := exploit.CheckHTTPHealth()
+			if err != nil {
+				util.Log.Errorf("exploit HTTP server is not running. err: %s", err)
+				util.Log.Errorf("Run go-exploitdb as server mode before reporting")
+				return subcommands.ExitFailure
+			}
+		}
+		dbclient, locked, err := report.NewDBClient(report.DBClientConf{
+			CveDictCnf:  c.Conf.CveDict,
+			OvalDictCnf: c.Conf.OvalDict,
+			GostCnf:     c.Conf.Gost,
+			ExploitCnf:  c.Conf.Exploit,
+			DebugSQL:    c.Conf.DebugSQL,
+		})
+		if locked {
+			util.Log.Errorf("SQLite3 is locked. Close other DB connections and try again: %s", err)
+			return subcommands.ExitFailure
+		}
+		if err != nil {
+			util.Log.Errorf("Failed to init DB Clients: %s", err)
+			return subcommands.ExitFailure
+		}
+		defer dbclient.CloseDB()
+
+		if res, err = report.FillCveInfos(*dbclient, res, dir); err != nil {
+			util.Log.Error(err)
+			return subcommands.ExitFailure
+		}
 	}
 
 	for _, w := range reports {
@@ -450,5 +414,6 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 			return subcommands.ExitFailure
 		}
 	}
+
 	return subcommands.ExitSuccess
 }

commands/scan.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -35,18 +35,8 @@ import (
 
 // ScanCmd is Subcommand of host discovery mode
 type ScanCmd struct {
-	debug          bool
 	configPath     string
-	resultsDir     string
-	logDir         string
-	cacheDBPath    string
-	httpProxy      string
 	askKeyPassword bool
-	containersOnly bool
-	deep           bool
-	skipBroken     bool
-	sshNative      bool
-	pipe           bool
 	timeoutSec     int
 	scanTimeoutSec int
 }
@@ -61,12 +51,12 @@ func (*ScanCmd) Synopsis() string { return "Scan vulnerabilities" }
 func (*ScanCmd) Usage() string {
 	return `scan:
 	scan
-		[-deep]
 		[-config=/path/to/config.toml]
 		[-results-dir=/path/to/results]
 		[-log-dir=/path/to/log]
 		[-cachedb-path=/path/to/cache.db]
 		[-ssh-native-insecure]
+		[-ssh-config]
 		[-containers-only]
 		[-skip-broken]
 		[-http-proxy=http://192.168.0.1:8080]
@@ -75,6 +65,7 @@ func (*ScanCmd) Usage() string {
 		[-timeout-scan=7200]
 		[-debug]
 		[-pipe]
+		[-vvv]
 
 		[SERVER]...
 `
@@ -82,93 +73,63 @@ func (*ScanCmd) Usage() string {
 
 // SetFlags set flag
 func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
-	f.BoolVar(&p.debug, "debug", false, "debug mode")
+	f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode")
 
 	wd, _ := os.Getwd()
-
 	defaultConfPath := filepath.Join(wd, "config.toml")
 	f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml")
 
 	defaultResultsDir := filepath.Join(wd, "results")
-	f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results")
+	f.StringVar(&c.Conf.ResultsDir, "results-dir", defaultResultsDir, "/path/to/results")
 
 	defaultLogDir := util.GetDefaultLogDir()
-	f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log")
+	f.StringVar(&c.Conf.LogDir, "log-dir", defaultLogDir, "/path/to/log")
 
 	defaultCacheDBPath := filepath.Join(wd, "cache.db")
-	f.StringVar(
-		&p.cacheDBPath,
-		"cachedb-path",
-		defaultCacheDBPath,
+	f.StringVar(&c.Conf.CacheDBPath, "cachedb-path", defaultCacheDBPath,
 		"/path/to/cache.db (local cache of changelog for Ubuntu/Debian)")
 
-	f.BoolVar(
-		&p.sshNative,
-		"ssh-native-insecure",
-		false,
+	f.BoolVar(&c.Conf.SSHNative, "ssh-native-insecure", false,
 		"Use Native Go implementation of SSH. Default: Use the external command")
 
-	f.BoolVar(
-		&p.containersOnly,
-		"containers-only",
-		false,
+	f.BoolVar(&c.Conf.SSHConfig, "ssh-config", false,
+		"Use SSH options specified in ssh_config preferentially")
+
+	f.BoolVar(&c.Conf.ContainersOnly, "containers-only", false,
 		"Scan containers only. Default: Scan both of hosts and containers")
 
-	f.BoolVar(
-		&p.skipBroken,
-		"skip-broken",
-		false,
+	f.BoolVar(&c.Conf.SkipBroken, "skip-broken", false,
 		"[For CentOS] yum update changelog with --skip-broken option")
 
-	f.StringVar(
-		&p.httpProxy,
-		"http-proxy",
-		"",
-		"http://proxy-url:port (default: empty)",
-	)
+	f.StringVar(&c.Conf.HTTPProxy, "http-proxy", "",
+		"http://proxy-url:port (default: empty)")
 
-	f.BoolVar(
-		&p.askKeyPassword,
-		"ask-key-password",
-		false,
+	f.BoolVar(&p.askKeyPassword, "ask-key-password", false,
 		"Ask ssh privatekey password before scanning",
 	)
 
-	f.BoolVar(
-		&p.deep,
-		"deep",
-		false,
-		"Deep scan mode. Scan accuracy improves and scanned information becomes richer. Since analysis of changelog, issue commands requiring sudo, but it may be slower and high load on the target server")
+	f.BoolVar(&c.Conf.Pipe, "pipe", false, "Use stdin via PIPE")
+	f.BoolVar(&c.Conf.Vvv, "vvv", false, "ssh -vvv")
 
-	f.BoolVar(
-		&p.pipe,
-		"pipe",
-		false,
-		"Use stdin via PIPE")
-
-	f.IntVar(
-		&p.timeoutSec,
-		"timeout",
-		5*60,
+	f.IntVar(&p.timeoutSec, "timeout", 5*60,
 		"Number of seconds for processing other than scan",
 	)
 
-	f.IntVar(
-		&p.scanTimeoutSec,
-		"timeout-scan",
-		120*60,
+	f.IntVar(&p.scanTimeoutSec, "timeout-scan", 120*60,
 		"Number of seconds for scanning vulnerabilities for all servers",
 	)
 }
 
 // Execute execute
 func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
-
 	// Setup Logger
-	c.Conf.Debug = p.debug
-	c.Conf.LogDir = p.logDir
 	util.Log = util.NewCustomLogger(c.ServerInfo{})
 
+	if err := mkdirDotVuls(); err != nil {
+		util.Log.Errorf("Failed to create .vuls: %s", err)
+		return subcommands.ExitUsageError
+	}
+
 	var keyPass string
 	var err error
 	if p.askKeyPassword {
@@ -190,7 +151,6 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 	util.Log.Info("Start scanning")
 	util.Log.Infof("config: %s", p.configPath)
 
-	c.Conf.Pipe = p.pipe
 	var servernames []string
 	if 0 < len(f.Args()) {
 		servernames = f.Args()
@@ -226,14 +186,6 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 	}
 	util.Log.Debugf("%s", pp.Sprintf("%v", target))
 
-	c.Conf.ResultsDir = p.resultsDir
-	c.Conf.CacheDBPath = p.cacheDBPath
-	c.Conf.SSHNative = p.sshNative
-	c.Conf.HTTPProxy = p.httpProxy
-	c.Conf.ContainersOnly = p.containersOnly
-	c.Conf.Deep = p.deep
-	c.Conf.SkipBroken = p.skipBroken
-
 	util.Log.Info("Validating config...")
 	if !c.Conf.ValidateOnScan() {
 		return subcommands.ExitUsageError
@@ -245,6 +197,12 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		return subcommands.ExitFailure
 	}
 
+	util.Log.Info("Checking Scan Modes... ")
+	if err := scan.CheckScanModes(); err != nil {
+		util.Log.Errorf("Fix config.toml: %s", err)
+		return subcommands.ExitFailure
+	}
+
 	util.Log.Info("Detecting Platforms... ")
 	scan.DetectPlatforms(p.timeoutSec)
 

commands/server.go

@@ -0,0 +1,239 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package commands
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	// "github.com/future-architect/vuls/Server"
+
+	c "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/exploit"
+	"github.com/future-architect/vuls/gost"
+	"github.com/future-architect/vuls/oval"
+	"github.com/future-architect/vuls/report"
+	"github.com/future-architect/vuls/server"
+	"github.com/future-architect/vuls/util"
+	"github.com/google/subcommands"
+	cvelog "github.com/kotakanbe/go-cve-dictionary/log"
+)
+
+// ServerCmd is subcommand for server
+type ServerCmd struct {
+	configPath  string
+	listen      string
+	cveDict     c.GoCveDictConf
+	ovalDict    c.GovalDictConf
+	gostConf    c.GostConf
+	exploitConf c.ExploitConf
+}
+
+// Name return subcommand name
+func (*ServerCmd) Name() string { return "server" }
+
+// Synopsis return synopsis
+func (*ServerCmd) Synopsis() string { return "Server" }
+
+// Usage return usage
+func (*ServerCmd) Usage() string {
+	return `Server:
+	Server
+		[-lang=en|ja]
+		[-config=/path/to/config.toml]
+		[-log-dir=/path/to/log]
+		[-cvss-over=7]
+		[-ignore-unscored-cves]
+		[-ignore-unfixed]
+		[-to-localfile]
+		[-format-json]
+		[-http-proxy=http://192.168.0.1:8080]
+		[-debug]
+		[-debug-sql]
+		[-listen=localhost:5515]
+		[-cvedb-type=sqlite3|mysql|postgres|redis|http]
+		[-cvedb-sqlite3-path=/path/to/cve.sqlite3]
+		[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
+		[-ovaldb-type=sqlite3|mysql|redis|http]
+		[-ovaldb-sqlite3-path=/path/to/oval.sqlite3]
+		[-ovaldb-url=http://127.0.0.1:1324 or DB connection string]
+		[-gostdb-type=sqlite3|mysql|redis|http]
+		[-gostdb-sqlite3-path=/path/to/gost.sqlite3]
+		[-gostdb-url=http://127.0.0.1:1325 or DB connection string]
+		[-exploitdb-type=sqlite3|mysql|redis|http]
+		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
+		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
+
+		[RFC3339 datetime format under results dir]
+`
+}
+
+// SetFlags set flag
+func (p *ServerCmd) SetFlags(f *flag.FlagSet) {
+	f.StringVar(&c.Conf.Lang, "lang", "en", "[en|ja]")
+	f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode")
+	f.BoolVar(&c.Conf.DebugSQL, "debug-sql", false, "SQL debug mode")
+
+	wd, _ := os.Getwd()
+	defaultConfPath := filepath.Join(wd, "config.toml")
+	f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml")
+
+	defaultResultsDir := filepath.Join(wd, "results")
+	f.StringVar(&c.Conf.ResultsDir, "results-dir", defaultResultsDir, "/path/to/results")
+
+	defaultLogDir := util.GetDefaultLogDir()
+	f.StringVar(&c.Conf.LogDir, "log-dir", defaultLogDir, "/path/to/log")
+
+	f.Float64Var(&c.Conf.CvssScoreOver, "cvss-over", 0,
+		"-cvss-over=6.5 means Servering CVSS Score 6.5 and over (default: 0 (means Server all))")
+
+	f.BoolVar(&c.Conf.IgnoreUnscoredCves, "ignore-unscored-cves", false,
+		"Don't Server the unscored CVEs")
+
+	f.BoolVar(&c.Conf.IgnoreUnfixed, "ignore-unfixed", false,
+		"Don't Server the unfixed CVEs")
+
+	f.StringVar(&c.Conf.HTTPProxy, "http-proxy", "",
+		"http://proxy-url:port (default: empty)")
+
+	f.BoolVar(&c.Conf.FormatJSON, "format-json", false, "JSON format")
+
+	f.BoolVar(&c.Conf.ToLocalFile, "to-localfile", false, "Write report to localfile")
+	f.StringVar(&p.listen, "listen", "localhost:5515",
+		"host:port (default: localhost:5515)")
+
+	f.StringVar(&p.cveDict.Type, "cvedb-type", "",
+		"DB type of go-cve-dictionary (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.cveDict.SQLite3Path, "cvedb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.cveDict.URL, "cvedb-url", "",
+		"http://go-cve-dictionary.com:1323 or DB connection string")
+
+	f.StringVar(&p.ovalDict.Type, "ovaldb-type", "",
+		"DB type of goval-dictionary (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.ovalDict.SQLite3Path, "ovaldb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.ovalDict.URL, "ovaldb-url", "",
+		"http://goval-dictionary.com:1324 or DB connection string")
+
+	f.StringVar(&p.gostConf.Type, "gostdb-type", "",
+		"DB type of gost (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.gostConf.SQLite3Path, "gostdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.gostConf.URL, "gostdb-url", "",
+		"http://gost.com:1325 or DB connection string")
+
+	f.StringVar(&p.exploitConf.Type, "exploitdb-type", "",
+		"DB type of exploit (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.exploitConf.SQLite3Path, "exploitdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.exploitConf.URL, "exploitdb-url", "",
+		"http://exploit.com:1326 or DB connection string")
+}
+
+// Execute execute
+func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
+	util.Log = util.NewCustomLogger(c.ServerInfo{})
+	cvelog.SetLogger(c.Conf.LogDir, false, c.Conf.Debug, false)
+
+	if err := c.Load(p.configPath, ""); err != nil {
+		util.Log.Errorf("Error loading %s, %s", p.configPath, err)
+		return subcommands.ExitUsageError
+	}
+
+	c.Conf.CveDict.Overwrite(p.cveDict)
+	c.Conf.OvalDict.Overwrite(p.ovalDict)
+	c.Conf.Gost.Overwrite(p.gostConf)
+	c.Conf.Exploit.Overwrite(p.exploitConf)
+
+	util.Log.Info("Validating config...")
+	if !c.Conf.ValidateOnReport() {
+		return subcommands.ExitUsageError
+	}
+
+	util.Log.Info("Validating db config...")
+	if !c.Conf.ValidateOnReportDB() {
+		return subcommands.ExitUsageError
+	}
+
+	if c.Conf.CveDict.URL != "" {
+		if err := report.CveClient.CheckHealth(); err != nil {
+			util.Log.Errorf("CVE HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with `-cvedb-type=sqlite3 -cvedb-sqlite3-path` option instead of -cvedb-url")
+			return subcommands.ExitFailure
+		}
+	}
+
+	if c.Conf.OvalDict.URL != "" {
+		err := oval.Base{}.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("OVAL HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run goval-dictionary as server mode before reporting or run with `-ovaldb-type=sqlite3 -ovaldb-sqlite3-path` option instead of -ovaldb-url")
+			return subcommands.ExitFailure
+		}
+	}
+
+	if c.Conf.Gost.URL != "" {
+		util.Log.Infof("gost: %s", c.Conf.Gost.URL)
+		err := gost.Base{}.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("gost HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run gost as server mode before reporting or run with `-gostdb-type=sqlite3 -gostdb-sqlite3-path` option instead of -gostdb-url")
+			return subcommands.ExitFailure
+		}
+	}
+
+	if c.Conf.Exploit.URL != "" {
+		err := exploit.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("exploit HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run go-exploitdb as server mode before reporting")
+			return subcommands.ExitFailure
+		}
+	}
+
+	dbclient, locked, err := report.NewDBClient(report.DBClientConf{
+		CveDictCnf:  c.Conf.CveDict,
+		OvalDictCnf: c.Conf.OvalDict,
+		GostCnf:     c.Conf.Gost,
+		ExploitCnf:  c.Conf.Exploit,
+		DebugSQL:    c.Conf.DebugSQL,
+	})
+	if locked {
+		util.Log.Errorf("SQLite3 is locked. Close other DB connections and try again: %s", err)
+		return subcommands.ExitFailure
+	}
+
+	if err != nil {
+		util.Log.Errorf("Failed to init DB Clients: %s", err)
+		return subcommands.ExitFailure
+	}
+
+	defer dbclient.CloseDB()
+
+	http.Handle("/vuls", server.VulsHandler{DBclient: *dbclient})
+	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "ok")
+	})
+	util.Log.Infof("Listening on %s", p.listen)
+	if err := http.ListenAndServe(p.listen, nil); err != nil {
+		util.Log.Errorf("Failed to start server: %s", err)
+		return subcommands.ExitFailure
+	}
+	return subcommands.ExitSuccess
+}

commands/tui.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -24,36 +24,23 @@ import (
 	"path/filepath"
 
 	c "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/exploit"
+	"github.com/future-architect/vuls/gost"
 	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/oval"
 	"github.com/future-architect/vuls/report"
 	"github.com/future-architect/vuls/util"
 	"github.com/google/subcommands"
+	cvelog "github.com/kotakanbe/go-cve-dictionary/log"
 )
 
 // TuiCmd is Subcommand of host discovery mode
 type TuiCmd struct {
-	lang       string
-	debugSQL   bool
-	debug      bool
-	configPath string
-	logDir     string
-
-	resultsDir string
-	refreshCve bool
-
-	cvedbtype        string
-	cvedbpath        string
-	cveDictionaryURL string
-
-	ovalDBType string
-	ovalDBPath string
-	ovalDBURL  string
-
-	cvssScoreOver      float64
-	ignoreUnscoredCves bool
-	ignoreUnfixed      bool
-
-	pipe bool
+	configPath  string
+	cveDict     c.GoCveDictConf
+	ovalDict    c.GovalDictConf
+	gostConf    c.GostConf
+	exploitConf c.ExploitConf
 }
 
 // Name return subcommand name
@@ -68,13 +55,8 @@ func (*TuiCmd) Usage() string {
 	tui
 		[-refresh-cve]
 		[-config=/path/to/config.toml]
-		[-cvedb-type=sqlite3|mysql|postgres]
-		[-cvedb-path=/path/to/cve.sqlite3]
-		[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
-		[-ovaldb-type=sqlite3|mysql]
-		[-ovaldb-path=/path/to/oval.sqlite3]
-		[-ovaldb-url=http://127.0.0.1:1324 or DB connection string]
 		[-cvss-over=7]
+		[-diff]
 		[-ignore-unscored-cves]
 		[-ignore-unfixed]
 		[-results-dir=/path/to/results]
@@ -82,6 +64,18 @@ func (*TuiCmd) Usage() string {
 		[-debug]
 		[-debug-sql]
 		[-pipe]
+		[-cvedb-type=sqlite3|mysql|postgres|redis|http]
+		[-cvedb-sqlite3-path=/path/to/cve.sqlite3]
+		[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
+		[-ovaldb-type=sqlite3|mysql|redis|http]
+		[-ovaldb-sqlite3-path=/path/to/oval.sqlite3]
+		[-ovaldb-url=http://127.0.0.1:1324 or DB connection string]
+		[-gostdb-type=sqlite3|mysql|redis|http]
+		[-gostdb-sqlite3-path=/path/to/gost.sqlite3]
+		[-gostdb-url=http://127.0.0.1:1325 or DB connection string]
+		[-exploitdb-type=sqlite3|mysql|redis|http]
+		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
+		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
 
 `
 }
@@ -89,86 +83,61 @@ func (*TuiCmd) Usage() string {
 // SetFlags set flag
 func (p *TuiCmd) SetFlags(f *flag.FlagSet) {
 	//  f.StringVar(&p.lang, "lang", "en", "[en|ja]")
-	f.BoolVar(&p.debugSQL, "debug-sql", false, "debug SQL")
-	f.BoolVar(&p.debug, "debug", false, "debug mode")
+	f.BoolVar(&c.Conf.DebugSQL, "debug-sql", false, "debug SQL")
+	f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode")
 
 	defaultLogDir := util.GetDefaultLogDir()
-	f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log")
+	f.StringVar(&c.Conf.LogDir, "log-dir", defaultLogDir, "/path/to/log")
 
 	wd, _ := os.Getwd()
 	defaultResultsDir := filepath.Join(wd, "results")
-	f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results")
+	f.StringVar(&c.Conf.ResultsDir, "results-dir", defaultResultsDir, "/path/to/results")
 
 	defaultConfPath := filepath.Join(wd, "config.toml")
 	f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml")
 
-	f.BoolVar(
-		&p.refreshCve,
-		"refresh-cve",
-		false,
+	f.BoolVar(&c.Conf.RefreshCve, "refresh-cve", false,
 		"Refresh CVE information in JSON file under results dir")
 
-	f.StringVar(
-		&p.cvedbtype,
-		"cvedb-type",
-		"sqlite3",
-		"DB type for fetching CVE dictionary (sqlite3, mysql or postgres)")
-
-	defaultCveDBPath := filepath.Join(wd, "cve.sqlite3")
-	f.StringVar(
-		&p.cvedbpath,
-		"cvedb-path",
-		defaultCveDBPath,
-		"/path/to/sqlite3 (For get cve detail from cve.sqlite3)")
-
-	f.StringVar(
-		&p.cveDictionaryURL,
-		"cvedb-url",
-		"",
-		"http://cve-dictionary.example.com:1323 or mysql connection string")
-
-	f.StringVar(
-		&p.ovalDBType,
-		"ovaldb-type",
-		"sqlite3",
-		"DB type for fetching OVAL dictionary (sqlite3 or mysql)")
-
-	defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3")
-	f.StringVar(
-		&p.ovalDBPath,
-		"ovaldb-path",
-		defaultOvalDBPath,
-		"/path/to/sqlite3 (For get oval detail from oval.sqlite3)")
-
-	f.StringVar(
-		&p.ovalDBURL,
-		"ovaldb-url",
-		"",
-		"http://goval-dictionary.example.com:1324 or mysql connection string")
-
-	f.Float64Var(
-		&p.cvssScoreOver,
-		"cvss-over",
-		0,
+	f.Float64Var(&c.Conf.CvssScoreOver, "cvss-over", 0,
 		"-cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))")
 
+	f.BoolVar(&c.Conf.Diff, "diff", false,
+		"Difference between previous result and current result ")
+
 	f.BoolVar(
-		&p.ignoreUnscoredCves,
-		"ignore-unscored-cves",
-		false,
+		&c.Conf.IgnoreUnscoredCves, "ignore-unscored-cves", false,
 		"Don't report the unscored CVEs")
 
-	f.BoolVar(
-		&p.ignoreUnfixed,
-		"ignore-unfixed",
-		false,
+	f.BoolVar(&c.Conf.IgnoreUnfixed, "ignore-unfixed", false,
 		"Don't report the unfixed CVEs")
 
-	f.BoolVar(
-		&p.pipe,
-		"pipe",
-		false,
-		"Use stdin via PIPE")
+	f.BoolVar(&c.Conf.Pipe, "pipe", false, "Use stdin via PIPE")
+
+	f.StringVar(&p.cveDict.Type, "cvedb-type", "",
+		"DB type of go-cve-dictionary (sqlite3, mysql, postgres or redis)")
+	f.StringVar(&p.cveDict.SQLite3Path, "cvedb-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.cveDict.URL, "cvedb-url", "",
+		"http://go-cve-dictionary.com:1323 or DB connection string")
+
+	f.StringVar(&p.ovalDict.Type, "ovaldb-type", "",
+		"DB type of goval-dictionary (sqlite3, mysql, postgres or redis)")
+	f.StringVar(&p.ovalDict.SQLite3Path, "ovaldb-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.ovalDict.URL, "ovaldb-url", "",
+		"http://goval-dictionary.com:1324 or DB connection string")
+
+	f.StringVar(&p.gostConf.Type, "gostdb-type", "",
+		"DB type of gost (sqlite3, mysql, postgres or redis)")
+	f.StringVar(&p.gostConf.SQLite3Path, "gostdb-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.gostConf.URL, "gostdb-url", "",
+		"http://gost.com:1325 or DB connection string")
+
+	f.StringVar(&p.exploitConf.Type, "exploitdb-type", "",
+		"DB type of exploit (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.exploitConf.SQLite3Path, "exploitdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.exploitConf.URL, "exploitdb-url", "",
+		"http://exploit.com:1326 or DB connection string")
+
 }
 
 // Execute execute
@@ -176,41 +145,36 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
 	c.Conf.Lang = "en"
 
 	// Setup Logger
-	c.Conf.Debug = p.debug
-	c.Conf.DebugSQL = p.debugSQL
-	c.Conf.LogDir = p.logDir
 	util.Log = util.NewCustomLogger(c.ServerInfo{})
-	log := util.Log
+	cvelog.SetLogger(c.Conf.LogDir, false, c.Conf.Debug, false)
 
 	if err := c.Load(p.configPath, ""); err != nil {
 		util.Log.Errorf("Error loading %s, %s", p.configPath, err)
 		return subcommands.ExitUsageError
 	}
 
-	c.Conf.ResultsDir = p.resultsDir
-	c.Conf.CveDBType = p.cvedbtype
-	c.Conf.CveDBPath = p.cvedbpath
-	c.Conf.CveDBURL = p.cveDictionaryURL
-	c.Conf.OvalDBType = p.ovalDBType
-	c.Conf.OvalDBPath = p.ovalDBPath
-	c.Conf.OvalDBURL = p.ovalDBURL
-	c.Conf.CvssScoreOver = p.cvssScoreOver
-	c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves
-	c.Conf.IgnoreUnfixed = p.ignoreUnfixed
-	c.Conf.RefreshCve = p.refreshCve
+	c.Conf.CveDict.Overwrite(p.cveDict)
+	c.Conf.OvalDict.Overwrite(p.ovalDict)
+	c.Conf.Gost.Overwrite(p.gostConf)
+	c.Conf.Exploit.Overwrite(p.exploitConf)
 
-	log.Info("Validating config...")
-	if !c.Conf.ValidateOnTui() {
-		return subcommands.ExitUsageError
+	var dir string
+	var err error
+	if c.Conf.Diff {
+		dir, err = report.JSONDir([]string{})
+	} else {
+		dir, err = report.JSONDir(f.Args())
 	}
-
-	c.Conf.Pipe = p.pipe
-
-	dir, err := report.JSONDir(f.Args())
 	if err != nil {
 		util.Log.Errorf("Failed to read from JSON: %s", err)
 		return subcommands.ExitFailure
 	}
+
+	util.Log.Info("Validating config...")
+	if !c.Conf.ValidateOnTui() {
+		return subcommands.ExitUsageError
+	}
+
 	var res models.ScanResults
 	if res, err = report.LoadScanResults(dir); err != nil {
 		util.Log.Error(err)
@@ -218,7 +182,66 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
 	}
 	util.Log.Infof("Loaded: %s", dir)
 
-	if res, err = report.FillCveInfos(res, dir); err != nil {
+	util.Log.Info("Validating db config...")
+	if !c.Conf.ValidateOnReportDB() {
+		return subcommands.ExitUsageError
+	}
+
+	if c.Conf.CveDict.URL != "" {
+		if err := report.CveClient.CheckHealth(); err != nil {
+			util.Log.Errorf("CVE HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with `-cvedb-type=sqlite3 -cvedb-sqlite3-path` option instead of -cvedb-url")
+			return subcommands.ExitFailure
+		}
+	}
+
+	if c.Conf.OvalDict.URL != "" {
+		err := oval.Base{}.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("OVAL HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run goval-dictionary as server mode before reporting or run with `-ovaldb-type=sqlite3 -ovaldb-sqlite3-path` option instead of -ovaldb-url")
+			return subcommands.ExitFailure
+		}
+	}
+
+	if c.Conf.Gost.URL != "" {
+		util.Log.Infof("gost: %s", c.Conf.Gost.URL)
+		err := gost.Base{}.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("gost HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run gost as server mode before reporting or run with `-gostdb-type=sqlite3 -gostdb-sqlite3-path` option instead of -gostdb-url")
+			return subcommands.ExitFailure
+		}
+	}
+
+	if c.Conf.Exploit.URL != "" {
+		err := exploit.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("exploit HTTP server is not running. err: %s", err)
+			util.Log.Errorf("Run go-exploitdb as server mode before reporting")
+			return subcommands.ExitFailure
+		}
+	}
+	dbclient, locked, err := report.NewDBClient(report.DBClientConf{
+		CveDictCnf:  c.Conf.CveDict,
+		OvalDictCnf: c.Conf.OvalDict,
+		GostCnf:     c.Conf.Gost,
+		ExploitCnf:  c.Conf.Exploit,
+		DebugSQL:    c.Conf.DebugSQL,
+	})
+	if locked {
+		util.Log.Errorf("SQLite3 is locked. Close other DB connections and try again: %s", err)
+		return subcommands.ExitFailure
+	}
+
+	if err != nil {
+		util.Log.Errorf("Failed to init DB Clients: %s", err)
+		return subcommands.ExitFailure
+	}
+
+	defer dbclient.CloseDB()
+
+	if res, err = report.FillCveInfos(*dbclient, res, dir); err != nil {
 		util.Log.Error(err)
 		return subcommands.ExitFailure
 	}

commands/util.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,8 +19,11 @@ package commands
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/howeyc/gopass"
+	homedir "github.com/mitchellh/go-homedir"
 )
 
 func getPasswd(prompt string) (string, error) {
@@ -31,8 +34,22 @@ func getPasswd(prompt string) (string, error) {
 			return "", fmt.Errorf("Failed to read password")
 		}
 		if 0 < len(pass) {
-			return string(pass[:]), nil
+			return string(pass), nil
 		}
 	}
 
 }
+
+func mkdirDotVuls() error {
+	home, err := homedir.Dir()
+	if err != nil {
+		return err
+	}
+	dotVuls := filepath.Join(home, ".vuls")
+	if _, err := os.Stat(dotVuls); os.IsNotExist(err) {
+		if err := os.Mkdir(dotVuls, 0700); err != nil {
+			return err
+		}
+	}
+	return nil
+}

commands/util_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

config/color.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

config/config_test.go

@@ -0,0 +1,103 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestSyslogConfValidate(t *testing.T) {
+	var tests = []struct {
+		conf              SyslogConf
+		expectedErrLength int
+	}{
+		{
+			conf:              SyslogConf{},
+			expectedErrLength: 0,
+		},
+		{
+			conf: SyslogConf{
+				Protocol: "tcp",
+				Port:     "5140",
+			},
+			expectedErrLength: 0,
+		},
+		{
+			conf: SyslogConf{
+				Protocol: "udp",
+				Port:     "12345",
+				Severity: "emerg",
+				Facility: "user",
+			},
+			expectedErrLength: 0,
+		},
+		{
+			conf: SyslogConf{
+				Protocol: "foo",
+				Port:     "514",
+			},
+			expectedErrLength: 1,
+		},
+		{
+			conf: SyslogConf{
+				Protocol: "invalid",
+				Port:     "-1",
+			},
+			expectedErrLength: 2,
+		},
+		{
+			conf: SyslogConf{
+				Protocol: "invalid",
+				Port:     "invalid",
+				Severity: "invalid",
+				Facility: "invalid",
+			},
+			expectedErrLength: 4,
+		},
+	}
+
+	for i, tt := range tests {
+		Conf.ToSyslog = true
+		errs := tt.conf.Validate()
+		if len(errs) != tt.expectedErrLength {
+			t.Errorf("test: %d, expected %d, actual %d", i, tt.expectedErrLength, len(errs))
+		}
+	}
+}
+
+func TestMajorVersion(t *testing.T) {
+	var tests = []struct {
+		in  Distro
+		out int
+	}{
+		{
+			in: Distro{
+				Family:  Amazon,
+				Release: "2 (2017.12)",
+			},
+			out: 2,
+		},
+		{
+			in: Distro{
+				Family:  Amazon,
+				Release: "2017.12",
+			},
+			out: 1,
+		},
+		{
+			in: Distro{
+				Family:  CentOS,
+				Release: "7.10",
+			},
+			out: 7,
+		},
+	}
+
+	for i, tt := range tests {
+		ver, err := tt.in.MajorVersion()
+		if err != nil {
+			t.Errorf("[%d] err occurred: %s", i, err)
+		}
+		if tt.out != ver {
+			t.Errorf("[%d] expected %d, actual %d", i, tt.out, ver)
+		}
+	}
+}

config/jsonloader.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

config/loader.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

config/tomlloader.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,30 +20,38 @@ package config
 import (
 	"fmt"
 	"os"
+	"regexp"
+	"strings"
 
 	"github.com/BurntSushi/toml"
-	"github.com/future-architect/vuls/contrib/owasp-dependency-check/parser"
-	log "github.com/sirupsen/logrus"
+	"github.com/knqyf263/go-cpe/naming"
 )
 
 // TOMLLoader loads config
 type TOMLLoader struct {
 }
 
-// Load load the configuraiton TOML file specified by path arg.
+// Load load the configuration TOML file specified by path arg.
 func (c TOMLLoader) Load(pathToToml, keyPass string) error {
-	if Conf.Debug {
-		log.SetLevel(log.DebugLevel)
-	}
-
 	var conf Config
 	if _, err := toml.DecodeFile(pathToToml, &conf); err != nil {
-		log.Error("Load config failed", err)
 		return err
 	}
-
 	Conf.EMail = conf.EMail
 	Conf.Slack = conf.Slack
+	Conf.Stride = conf.Stride
+	Conf.HipChat = conf.HipChat
+	Conf.ChatWork = conf.ChatWork
+	Conf.Saas = conf.Saas
+	Conf.Syslog = conf.Syslog
+	Conf.HTTP = conf.HTTP
+	Conf.AWS = conf.AWS
+	Conf.Azure = conf.Azure
+
+	Conf.CveDict = conf.CveDict
+	Conf.OvalDict = conf.OvalDict
+	Conf.Gost = conf.Gost
+	Conf.Exploit = conf.Exploit
 
 	d := conf.Default
 	Conf.Default = d
@@ -54,17 +62,16 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
 	}
 
 	i := 0
-	for name, v := range conf.Servers {
+	for serverName, v := range conf.Servers {
 		if 0 < len(v.KeyPassword) {
-			log.Warn("[Deprecated] KEYPASSWORD IN CONFIG FILE ARE UNSECURE. REMOVE THEM IMMEDIATELY FOR A SECURITY REASONS. THEY WILL BE REMOVED IN A FUTURE RELEASE.")
+			return fmt.Errorf("[Deprecated] KEYPASSWORD IN CONFIG FILE ARE UNSECURE. REMOVE THEM IMMEDIATELY FOR A SECURITY REASONS. THEY WILL BE REMOVED IN A FUTURE RELEASE: %s", serverName)
 		}
 
-		s := ServerInfo{ServerName: name}
-
+		s := ServerInfo{ServerName: serverName}
 		if v.Type != ServerTypePseudo {
 			s.Host = v.Host
 			if len(s.Host) == 0 {
-				return fmt.Errorf("%s is invalid. host is empty", name)
+				return fmt.Errorf("%s is invalid. host is empty", serverName)
 			}
 
 			switch {
@@ -83,7 +90,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
 				s.User = d.User
 			default:
 				if s.Port != "local" {
-					return fmt.Errorf("%s is invalid. User is empty", name)
+					return fmt.Errorf("%s is invalid. User is empty", serverName)
 				}
 			}
 
@@ -94,42 +101,87 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
 			if s.KeyPath != "" {
 				if _, err := os.Stat(s.KeyPath); err != nil {
 					return fmt.Errorf(
-						"%s is invalid. keypath: %s not exists", name, s.KeyPath)
+						"%s is invalid. keypath: %s not exists", serverName, s.KeyPath)
 				}
 			}
 
-			//  s.KeyPassword = keyPass
 			s.KeyPassword = v.KeyPassword
 			if len(s.KeyPassword) == 0 {
 				s.KeyPassword = d.KeyPassword
 			}
 		}
 
+		s.ScanMode = v.ScanMode
+		if len(s.ScanMode) == 0 {
+			s.ScanMode = d.ScanMode
+			if len(s.ScanMode) == 0 {
+				s.ScanMode = []string{"fast"}
+			}
+		}
+		for _, m := range s.ScanMode {
+			switch m {
+			case "fast":
+				s.Mode.Set(Fast)
+			case "fast-root":
+				s.Mode.Set(FastRoot)
+			case "deep":
+				s.Mode.Set(Deep)
+			case "offline":
+				s.Mode.Set(Offline)
+			default:
+				return fmt.Errorf("scanMode: %s of %s is invalie. Specify -fast, -fast-root, -deep or offline", m, serverName)
+			}
+		}
+		if err := s.Mode.validate(); err != nil {
+			return fmt.Errorf("%s in %s", err, serverName)
+		}
+
 		s.CpeNames = v.CpeNames
 		if len(s.CpeNames) == 0 {
 			s.CpeNames = d.CpeNames
 		}
 
-		s.DependencyCheckXMLPath = v.DependencyCheckXMLPath
-		if len(s.DependencyCheckXMLPath) == 0 {
-			s.DependencyCheckXMLPath = d.DependencyCheckXMLPath
+		for i, n := range s.CpeNames {
+			uri, err := toCpeURI(n)
+			if err != nil {
+				return fmt.Errorf("Failed to parse CPENames %s in %s: %s", n, serverName, err)
+			}
+			s.CpeNames[i] = uri
 		}
 
-		// Load CPEs from OWASP Dependency Check XML
-		if len(s.DependencyCheckXMLPath) != 0 {
-			cpes, err := parser.Parse(s.DependencyCheckXMLPath)
-			if err != nil {
-				return fmt.Errorf(
-					"Failed to read OWASP Dependency Check XML: %s", err)
-			}
-			log.Debugf("Loaded from OWASP Dependency Check XML: %s",
-				s.ServerName)
-			s.CpeNames = append(s.CpeNames, cpes...)
+		s.ContainersIncluded = v.ContainersIncluded
+		if len(s.ContainersIncluded) == 0 {
+			s.ContainersIncluded = d.ContainersIncluded
+		}
+
+		s.ContainersExcluded = v.ContainersExcluded
+		if len(s.ContainersExcluded) == 0 {
+			s.ContainersExcluded = d.ContainersExcluded
+		}
+
+		s.ContainerType = v.ContainerType
+		if len(s.ContainerType) == 0 {
+			s.ContainerType = d.ContainerType
 		}
 
 		s.Containers = v.Containers
-		if len(s.Containers.Includes) == 0 {
-			s.Containers = d.Containers
+		for contName, cont := range s.Containers {
+			cont.IgnoreCves = append(cont.IgnoreCves, d.IgnoreCves...)
+			s.Containers[contName] = cont
+		}
+
+		if len(v.DependencyCheckXMLPath) != 0 || len(d.DependencyCheckXMLPath) != 0 {
+			return fmt.Errorf("[DEPRECATED] dependencyCheckXMLPath IS DEPRECATED. USE owaspDCXMLPath INSTEAD: %s", serverName)
+		}
+
+		s.OwaspDCXMLPath = v.OwaspDCXMLPath
+		if len(s.OwaspDCXMLPath) == 0 {
+			s.OwaspDCXMLPath = d.OwaspDCXMLPath
+		}
+
+		s.Memo = v.Memo
+		if s.Memo == "" {
+			s.Memo = d.Memo
 		}
 
 		s.IgnoreCves = v.IgnoreCves
@@ -146,19 +198,43 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
 			}
 		}
 
-		s.Optional = v.Optional
-		for _, dkv := range d.Optional {
+		s.IgnorePkgsRegexp = v.IgnorePkgsRegexp
+		for _, pkg := range d.IgnorePkgsRegexp {
 			found := false
-			for _, kv := range s.Optional {
-				if dkv[0] == kv[0] {
+			for _, p := range s.IgnorePkgsRegexp {
+				if pkg == p {
 					found = true
 					break
 				}
 			}
 			if !found {
-				s.Optional = append(s.Optional, dkv)
+				s.IgnorePkgsRegexp = append(s.IgnorePkgsRegexp, pkg)
 			}
 		}
+		for _, reg := range s.IgnorePkgsRegexp {
+			_, err := regexp.Compile(reg)
+			if err != nil {
+				return fmt.Errorf("Faild to parse %s in %s. err: %s", reg, serverName, err)
+			}
+		}
+		for contName, cont := range s.Containers {
+			for _, reg := range cont.IgnorePkgsRegexp {
+				_, err := regexp.Compile(reg)
+				if err != nil {
+					return fmt.Errorf("Faild to parse %s in %s@%s. err: %s",
+						reg, contName, serverName, err)
+				}
+			}
+		}
+
+		opt := map[string]interface{}{}
+		for k, v := range d.Optional {
+			opt[k] = v
+		}
+		for k, v := range v.Optional {
+			opt[k] = v
+		}
+		s.Optional = opt
 
 		s.Enablerepo = v.Enablerepo
 		if len(s.Enablerepo) == 0 {
@@ -172,18 +248,36 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
 				default:
 					return fmt.Errorf(
 						"For now, enablerepo have to be base or updates: %s, servername: %s",
-						s.Enablerepo, name)
+						s.Enablerepo, serverName)
 				}
 			}
 		}
 
+		s.UUIDs = v.UUIDs
 		s.Type = v.Type
 
 		s.LogMsgAnsiColor = Colors[i%len(Colors)]
 		i++
 
-		servers[name] = s
+		servers[serverName] = s
 	}
 	Conf.Servers = servers
 	return nil
 }
+
+func toCpeURI(cpename string) (string, error) {
+	if strings.HasPrefix(cpename, "cpe:2.3:") {
+		wfn, err := naming.UnbindFS(cpename)
+		if err != nil {
+			return "", err
+		}
+		return naming.BindToURI(wfn), nil
+	} else if strings.HasPrefix(cpename, "cpe:/") {
+		wfn, err := naming.UnbindURI(cpename)
+		if err != nil {
+			return "", err
+		}
+		return naming.BindToURI(wfn), nil
+	}
+	return "", fmt.Errorf("Unknow CPE format: %s", cpename)
+}

config/tomlloader_test.go

@@ -0,0 +1,44 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestToCpeURI(t *testing.T) {
+	var tests = []struct {
+		in       string
+		expected string
+		err      bool
+	}{
+		{
+			in:       "",
+			expected: "",
+			err:      true,
+		},
+		{
+			in:       "cpe:/a:microsoft:internet_explorer:10",
+			expected: "cpe:/a:microsoft:internet_explorer:10",
+			err:      false,
+		},
+		{
+			in:       "cpe:2.3:a:microsoft:internet_explorer:10:*:*:*:*:*:*:*",
+			expected: "cpe:/a:microsoft:internet_explorer:10",
+			err:      false,
+		},
+	}
+
+	for i, tt := range tests {
+		actual, err := toCpeURI(tt.in)
+		if err != nil && !tt.err {
+			t.Errorf("[%d] unexpected error occurred, in: %s act: %s, exp: %s",
+				i, tt.in, actual, tt.expected)
+		} else if err == nil && tt.err {
+			t.Errorf("[%d] expected error is not occurred, in: %s act: %s, exp: %s",
+				i, tt.in, actual, tt.expected)
+		}
+		if actual != tt.expected {
+			t.Errorf("[%d] in: %s, actual: %s, expected: %s",
+				i, tt.in, actual, tt.expected)
+		}
+	}
+}

contrib/owasp-dependency-check/parser/parser.go

@@ -6,6 +6,8 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type analysis struct {
@@ -30,17 +32,19 @@ func appendIfMissing(slice []string, str string) []string {
 	return append(slice, str)
 }
 
-// Parse parses XML and collect list of cpe
+// Parse parses OWASP dependency check XML and collect list of cpe
 func Parse(path string) ([]string, error) {
 	file, err := os.Open(path)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to open: %s", err)
+		log.Warnf("OWASP Dependency Check XML is not found: %s", path)
+		return []string{}, nil
 	}
 	defer file.Close()
 
 	b, err := ioutil.ReadAll(file)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to read: %s", err)
+		log.Warnf("Failed to read OWASP Dependency Check XML: %s", path)
+		return []string{}, nil
 	}
 
 	var anal analysis

cwe/owasp.go

@@ -0,0 +1,65 @@
+package cwe
+
+// OwaspTopTen2017 has CWE-ID in OWSP Top 10
+var OwaspTopTen2017 = map[string]string{
+	"77":  "1",
+	"89":  "1",
+	"564": "1",
+	"917": "1",
+
+	"287": "2",
+	"384": "2",
+
+	"220": "3",
+	"310": "3",
+	"312": "3",
+	"319": "3",
+	"326": "3",
+	"359": "3",
+
+	"611": "4",
+
+	"22":  "5",
+	"284": "5",
+	"285": "5",
+	"639": "5",
+
+	"2":   "6",
+	"16":  "6",
+	"388": "6",
+
+	"79": "7",
+
+	"502": "8",
+
+	"223": "10",
+	"778": "10",
+}
+
+// OwaspTopTen2017GitHubURLEn has GitHub links
+var OwaspTopTen2017GitHubURLEn = map[string]string{
+	"1":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa1-injection.md",
+	"2":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa2-broken-authentication.md",
+	"3":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa3-sensitive-data-disclosure.md",
+	"4":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa4-xxe.md",
+	"5":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa5-broken-access-control.md",
+	"6":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa6-security-misconfiguration.md",
+	"7":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa7-xss.md",
+	"8":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa8-insecure-deserialization.md",
+	"9":  "https://github.com/OWASP/Top10/blob/master/2017/en/0xa9-known-vulns.md<Paste>",
+	"10": "https://github.com/OWASP/Top10/blob/master/2017/en/0xaa-logging-detection-response.md",
+}
+
+// OwaspTopTen2017GitHubURLJa has GitHub links
+var OwaspTopTen2017GitHubURLJa = map[string]string{
+	"1":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa1-injection.md",
+	"2":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa2-broken-authentication.md",
+	"3":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa3-sensitive-data-disclosure.md",
+	"4":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa4-xxe.md",
+	"5":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa5-broken-access-control.md",
+	"6":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa6-security-misconfiguration.md",
+	"7":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa7-xss.md",
+	"8":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa8-insecure-deserialization.md",
+	"9":  "https://github.com/OWASP/Top10/blob/master/2017/ja/0xa9-known-vulns.md<Paste>",
+	"10": "https://github.com/OWASP/Top10/blob/master/2017/ja/0xaa-logging-detection-response.md",
+}

exploit/exploit.go

@@ -0,0 +1,135 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package exploit
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	cnf "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+	"github.com/mozqnet/go-exploitdb/db"
+	exploitmodels "github.com/mozqnet/go-exploitdb/models"
+	"github.com/parnurzeal/gorequest"
+)
+
+// FillWithExploit fills exploit information that has in Exploit
+func FillWithExploit(driver db.DB, r *models.ScanResult) (nExploitCve int, err error) {
+	if cnf.Conf.Exploit.IsFetchViaHTTP() {
+		var cveIDs []string
+		for cveID := range r.ScannedCves {
+			cveIDs = append(cveIDs, cveID)
+		}
+		prefix, _ := util.URLPathJoin(cnf.Conf.Exploit.URL, "cves")
+		responses, err := getCvesViaHTTP(cveIDs, prefix)
+		if err != nil {
+			return 0, err
+		}
+		for _, res := range responses {
+			exps := []*exploitmodels.Exploit{}
+			if err := json.Unmarshal([]byte(res.json), &exps); err != nil {
+				return 0, err
+			}
+			exploits := convertToModels(exps)
+			v, ok := r.ScannedCves[res.request.cveID]
+			if ok {
+				v.Exploits = exploits
+			}
+			r.ScannedCves[res.request.cveID] = v
+			nExploitCve++
+		}
+	} else {
+		if driver == nil {
+			return 0, nil
+		}
+		for cveID, vuln := range r.ScannedCves {
+			es := driver.GetExploitByCveID(cveID)
+			if len(es) == 0 {
+				continue
+			}
+			exploits := convertToModels(es)
+			vuln.Exploits = exploits
+			r.ScannedCves[cveID] = vuln
+			nExploitCve++
+		}
+	}
+	return nExploitCve, nil
+}
+
+// convertToModels converts gost model to vuls model
+func convertToModels(es []*exploitmodels.Exploit) (exploits []models.Exploit) {
+	for _, e := range es {
+		var documentURL, paperURL, shellURL *string
+		if e.OffensiveSecurity != nil {
+			os := e.OffensiveSecurity
+			if os.Document != nil {
+				documentURL = &os.Document.DocumentURL
+			}
+			if os.ShellCode != nil {
+				shellURL = &os.ShellCode.ShellCodeURL
+			}
+			if os.Paper != nil {
+				paperURL = &os.Paper.PaperURL
+			}
+		}
+		exploit := models.Exploit{
+			ExploitType:  e.ExploitType,
+			ID:           e.ExploitUniqueID,
+			URL:          e.URL,
+			Description:  e.Description,
+			DocumentURL:  documentURL,
+			ShellCodeURL: shellURL,
+			PaperURL:     paperURL,
+		}
+		exploits = append(exploits, exploit)
+	}
+	return exploits
+}
+
+// CheckHTTPHealth do health check
+func CheckHTTPHealth() error {
+	if !cnf.Conf.Exploit.IsFetchViaHTTP() {
+		return nil
+	}
+
+	url := fmt.Sprintf("%s/health", cnf.Conf.Exploit.URL)
+	var errs []error
+	var resp *http.Response
+	resp, _, errs = gorequest.New().Get(url).End()
+	//  resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
+	//  resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End()
+	if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
+		return fmt.Errorf("Failed to connect to exploit server. url: %s, errs: %v",
+			url, errs)
+	}
+	return nil
+}
+
+// CheckIfExploitFetched checks if oval entries are in DB by family, release.
+func CheckIfExploitFetched(driver db.DB, osFamily string) (fetched bool, err error) {
+	//TODO
+	return true, nil
+}
+
+// CheckIfExploitFresh checks if oval entries are fresh enough
+func CheckIfExploitFresh(driver db.DB, osFamily string) (ok bool, err error) {
+	//TODO
+	return true, nil
+}

exploit/exploit_test.go

@@ -0,0 +1,8 @@
+package exploit
+
+import (
+	"testing"
+)
+
+func TestSetPackageStates(t *testing.T) {
+}

exploit/util.go

@@ -0,0 +1,133 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package exploit
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/cenkalti/backoff"
+	"github.com/future-architect/vuls/util"
+	"github.com/parnurzeal/gorequest"
+)
+
+type response struct {
+	request request
+	json    string
+}
+
+func getCvesViaHTTP(cveIDs []string, urlPrefix string) (
+	responses []response, err error) {
+	nReq := len(cveIDs)
+	reqChan := make(chan request, nReq)
+	resChan := make(chan response, nReq)
+	errChan := make(chan error, nReq)
+	defer close(reqChan)
+	defer close(resChan)
+	defer close(errChan)
+
+	go func() {
+		for _, cveID := range cveIDs {
+			reqChan <- request{
+				cveID: cveID,
+			}
+		}
+	}()
+
+	concurrency := 10
+	tasks := util.GenWorkers(concurrency)
+	for i := 0; i < nReq; i++ {
+		tasks <- func() {
+			select {
+			case req := <-reqChan:
+				url, err := util.URLPathJoin(
+					urlPrefix,
+					req.cveID,
+				)
+				if err != nil {
+					errChan <- err
+				} else {
+					util.Log.Debugf("HTTP Request to %s", url)
+					httpGet(url, req, resChan, errChan)
+				}
+			}
+		}
+	}
+
+	timeout := time.After(2 * 60 * time.Second)
+	var errs []error
+	for i := 0; i < nReq; i++ {
+		select {
+		case res := <-resChan:
+			responses = append(responses, res)
+		case err := <-errChan:
+			errs = append(errs, err)
+		case <-timeout:
+			return nil, fmt.Errorf("Timeout Fetching OVAL")
+		}
+	}
+	if len(errs) != 0 {
+		return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
+	}
+	return
+}
+
+type request struct {
+	osMajorVersion string
+	packName       string
+	isSrcPack      bool
+	cveID          string
+}
+
+func httpGet(url string, req request, resChan chan<- response, errChan chan<- error) {
+	var body string
+	var errs []error
+	var resp *http.Response
+	count, retryMax := 0, 3
+	f := func() (err error) {
+		//  resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
+		resp, body, errs = gorequest.New().Get(url).End()
+		if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
+			count++
+			if count == retryMax {
+				return nil
+			}
+			return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
+				errs, url, resp)
+		}
+		return nil
+	}
+	notify := func(err error, t time.Duration) {
+		util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
+	}
+	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
+	if err != nil {
+		errChan <- fmt.Errorf("HTTP Error %s", err)
+		return
+	}
+	if count == retryMax {
+		errChan <- fmt.Errorf("HRetry count exceeded")
+		return
+	}
+
+	resChan <- response{
+		request: req,
+		json:    body,
+	}
+}

gost/debian.go

@@ -0,0 +1,182 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gost
+
+import (
+	"encoding/json"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+	"github.com/knqyf263/gost/db"
+	gostmodels "github.com/knqyf263/gost/models"
+)
+
+// Debian is Gost client for Debian GNU/Linux
+type Debian struct {
+	Base
+}
+
+type packCves struct {
+	packName  string
+	isSrcPack bool
+	cves      []models.CveContent
+}
+
+// FillWithGost fills cve information that has in Gost
+func (deb Debian) FillWithGost(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
+	linuxImage := "linux-image-" + r.RunningKernel.Release
+	// Add linux and set the version of running kernel to search OVAL.
+	if r.Container.ContainerID == "" {
+		newVer := ""
+		if p, ok := r.Packages[linuxImage]; ok {
+			newVer = p.NewVersion
+		}
+		r.Packages["linux"] = models.Package{
+			Name:       "linux",
+			Version:    r.RunningKernel.Version,
+			NewVersion: newVer,
+		}
+	}
+
+	packCvesList := []packCves{}
+	if config.Conf.Gost.IsFetchViaHTTP() {
+		url, _ := util.URLPathJoin(config.Conf.Gost.URL, "debian", major(r.Release), "pkgs")
+		responses, err := getAllUnfixedCvesViaHTTP(r, url)
+		if err != nil {
+			return 0, err
+		}
+
+		for _, res := range responses {
+			debCves := map[string]gostmodels.DebianCVE{}
+			if err := json.Unmarshal([]byte(res.json), &debCves); err != nil {
+				return 0, err
+			}
+			cves := []models.CveContent{}
+			for _, debcve := range debCves {
+				cves = append(cves, *deb.ConvertToModel(&debcve))
+			}
+			packCvesList = append(packCvesList, packCves{
+				packName:  res.request.packName,
+				isSrcPack: res.request.isSrcPack,
+				cves:      cves,
+			})
+		}
+	} else {
+		if driver == nil {
+			return 0, nil
+		}
+		for _, pack := range r.Packages {
+			cveDebs := driver.GetUnfixedCvesDebian(major(r.Release), pack.Name)
+			cves := []models.CveContent{}
+			for _, cveDeb := range cveDebs {
+				cves = append(cves, *deb.ConvertToModel(&cveDeb))
+			}
+			packCvesList = append(packCvesList, packCves{
+				packName:  pack.Name,
+				isSrcPack: false,
+				cves:      cves,
+			})
+		}
+
+		// SrcPack
+		for _, pack := range r.SrcPackages {
+			cveDebs := driver.GetUnfixedCvesDebian(major(r.Release), pack.Name)
+			cves := []models.CveContent{}
+			for _, cveDeb := range cveDebs {
+				cves = append(cves, *deb.ConvertToModel(&cveDeb))
+			}
+			packCvesList = append(packCvesList, packCves{
+				packName:  pack.Name,
+				isSrcPack: true,
+				cves:      cves,
+			})
+		}
+	}
+
+	delete(r.Packages, "linux")
+
+	for _, p := range packCvesList {
+		for _, cve := range p.cves {
+			v, ok := r.ScannedCves[cve.CveID]
+			if ok {
+				if v.CveContents == nil {
+					v.CveContents = models.NewCveContents(cve)
+				} else {
+					v.CveContents[models.DebianSecurityTracker] = cve
+				}
+			} else {
+				v = models.VulnInfo{
+					CveID:       cve.CveID,
+					CveContents: models.NewCveContents(cve),
+					Confidences: models.Confidences{models.DebianSecurityTrackerMatch},
+				}
+				nCVEs++
+			}
+
+			names := []string{}
+			if p.isSrcPack {
+				if srcPack, ok := r.SrcPackages[p.packName]; ok {
+					for _, binName := range srcPack.BinaryNames {
+						if _, ok := r.Packages[binName]; ok {
+							names = append(names, binName)
+						}
+					}
+				}
+			} else {
+				if p.packName == "linux" {
+					names = append(names, linuxImage)
+				} else {
+					names = append(names, p.packName)
+				}
+			}
+
+			for _, name := range names {
+				v.AffectedPackages = v.AffectedPackages.Store(models.PackageStatus{
+					Name:        name,
+					FixState:    "open",
+					NotFixedYet: true,
+				})
+			}
+			r.ScannedCves[cve.CveID] = v
+		}
+	}
+	return nCVEs, nil
+}
+
+// ConvertToModel converts gost model to vuls model
+func (deb Debian) ConvertToModel(cve *gostmodels.DebianCVE) *models.CveContent {
+	severity := ""
+	for _, p := range cve.Package {
+		for _, r := range p.Release {
+			severity = r.Urgency
+			break
+		}
+	}
+	return &models.CveContent{
+		Type:          models.DebianSecurityTracker,
+		CveID:         cve.CveID,
+		Summary:       cve.Description,
+		Cvss2Severity: severity,
+		Cvss3Severity: severity,
+		SourceLink:    "https://security-tracker.debian.org/tracker/" + cve.CveID,
+		Optional: map[string]string{
+			"attack range": cve.Scope,
+		},
+	}
+}

gost/gost.go

@@ -0,0 +1,104 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gost
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	cnf "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/knqyf263/gost/db"
+	"github.com/parnurzeal/gorequest"
+)
+
+// Client is the interface of OVAL client.
+type Client interface {
+	FillWithGost(db.DB, *models.ScanResult) (int, error)
+
+	//TODO implement
+	// CheckHTTPHealth() error
+	// CheckIfGostFetched checks if Gost entries are fetched
+	// CheckIfGostFetched(db.DB, string, string) (bool, error)
+	// CheckIfGostFresh(db.DB, string, string) (bool, error)
+}
+
+// NewClient make Client by family
+func NewClient(family string) Client {
+	switch family {
+	case cnf.RedHat, cnf.CentOS:
+		return RedHat{}
+	case cnf.Debian:
+		return Debian{}
+	case cnf.Windows:
+		return Microsoft{}
+	default:
+		return Pseudo{}
+	}
+}
+
+// Base is a base struct
+type Base struct {
+	family string
+}
+
+// CheckHTTPHealth do health check
+func (b Base) CheckHTTPHealth() error {
+	if !cnf.Conf.Gost.IsFetchViaHTTP() {
+		return nil
+	}
+
+	url := fmt.Sprintf("%s/health", cnf.Conf.Gost.URL)
+	var errs []error
+	var resp *http.Response
+	resp, _, errs = gorequest.New().Get(url).End()
+	//  resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
+	//  resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End()
+	if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
+		return fmt.Errorf("Failed to connect to gost server. url: %s, errs: %v",
+			url, errs)
+	}
+	return nil
+}
+
+// CheckIfGostFetched checks if oval entries are in DB by family, release.
+func (b Base) CheckIfGostFetched(driver db.DB, osFamily string) (fetched bool, err error) {
+	//TODO
+	return true, nil
+}
+
+// CheckIfGostFresh checks if oval entries are fresh enough
+func (b Base) CheckIfGostFresh(driver db.DB, osFamily string) (ok bool, err error) {
+	//TODO
+	return true, nil
+}
+
+// Pseudo is Gost client except for RedHat family and Debian
+type Pseudo struct {
+	Base
+}
+
+// FillWithGost fills cve information that has in Gost
+func (pse Pseudo) FillWithGost(driver db.DB, r *models.ScanResult) (int, error) {
+	return 0, nil
+}
+
+func major(osVer string) (majorVersion string) {
+	return strings.Split(osVer, ".")[0]
+}

gost/gost_test.go

@@ -0,0 +1,129 @@
+package gost
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/future-architect/vuls/models"
+	gostmodels "github.com/knqyf263/gost/models"
+)
+
+func TestSetPackageStates(t *testing.T) {
+	var tests = []struct {
+		pkgstats  []gostmodels.RedhatPackageState
+		installed models.Packages
+		release   string
+		in        models.VulnInfo
+		out       models.PackageStatuses
+	}{
+
+		//0 one
+		{
+			pkgstats: []gostmodels.RedhatPackageState{
+				{
+					FixState:    "Will not fix",
+					PackageName: "bouncycastle",
+					Cpe:         "cpe:/o:redhat:enterprise_linux:7",
+				},
+			},
+			installed: models.Packages{
+				"bouncycastle": models.Package{},
+			},
+			release: "7",
+			in:      models.VulnInfo{},
+			out: []models.PackageStatus{
+				{
+					Name:        "bouncycastle",
+					FixState:    "Will not fix",
+					NotFixedYet: true,
+				},
+			},
+		},
+
+		//1 two
+		{
+			pkgstats: []gostmodels.RedhatPackageState{
+				{
+					FixState:    "Will not fix",
+					PackageName: "bouncycastle",
+					Cpe:         "cpe:/o:redhat:enterprise_linux:7",
+				},
+				{
+					FixState:    "Fix deferred",
+					PackageName: "pack_a",
+					Cpe:         "cpe:/o:redhat:enterprise_linux:7",
+				},
+				// ignore not-installed-package
+				{
+					FixState:    "Fix deferred",
+					PackageName: "pack_b",
+					Cpe:         "cpe:/o:redhat:enterprise_linux:7",
+				},
+			},
+			installed: models.Packages{
+				"bouncycastle": models.Package{},
+				"pack_a":       models.Package{},
+			},
+			release: "7",
+			in:      models.VulnInfo{},
+			out: []models.PackageStatus{
+				{
+					Name:        "bouncycastle",
+					FixState:    "Will not fix",
+					NotFixedYet: true,
+				},
+				{
+					Name:        "pack_a",
+					FixState:    "Fix deferred",
+					NotFixedYet: true,
+				},
+			},
+		},
+
+		//2 ignore affected
+		{
+			pkgstats: []gostmodels.RedhatPackageState{
+				{
+					FixState:    "affected",
+					PackageName: "bouncycastle",
+					Cpe:         "cpe:/o:redhat:enterprise_linux:7",
+				},
+			},
+			installed: models.Packages{
+				"bouncycastle": models.Package{},
+			},
+			release: "7",
+			in: models.VulnInfo{
+				AffectedPackages: models.PackageStatuses{},
+			},
+			out: models.PackageStatuses{},
+		},
+
+		//3 look only the same os release.
+		{
+			pkgstats: []gostmodels.RedhatPackageState{
+				{
+					FixState:    "Will not fix",
+					PackageName: "bouncycastle",
+					Cpe:         "cpe:/o:redhat:enterprise_linux:6",
+				},
+			},
+			installed: models.Packages{
+				"bouncycastle": models.Package{},
+			},
+			release: "7",
+			in: models.VulnInfo{
+				AffectedPackages: models.PackageStatuses{},
+			},
+			out: models.PackageStatuses{},
+		},
+	}
+
+	r := RedHat{}
+	for i, tt := range tests {
+		out := r.mergePackageStates(tt.in, tt.pkgstats, tt.installed, tt.release)
+		if ok := reflect.DeepEqual(tt.out, out); !ok {
+			t.Errorf("[%d]\nexpected: %v:%T\n  actual: %v:%T\n", i, tt.out, tt.out, out, out)
+		}
+	}
+}

gost/microsoft.go

@@ -0,0 +1,116 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Corporation , Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gost
+
+import (
+	"strings"
+
+	"github.com/future-architect/vuls/models"
+	"github.com/knqyf263/gost/db"
+	gostmodels "github.com/knqyf263/gost/models"
+)
+
+// Microsoft is Gost client for windows
+type Microsoft struct {
+	Base
+}
+
+// FillWithGost fills cve information that has in Gost
+func (ms Microsoft) FillWithGost(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
+	if driver == nil {
+		return 0, nil
+	}
+	var cveIDs []string
+	for cveID := range r.ScannedCves {
+		cveIDs = append(cveIDs, cveID)
+	}
+	for cveID, msCve := range driver.GetMicrosoftMulti(cveIDs) {
+		if _, ok := r.ScannedCves[cveID]; !ok {
+			continue
+		}
+		cveCont := ms.ConvertToModel(&msCve)
+		v, _ := r.ScannedCves[cveID]
+		if v.CveContents == nil {
+			v.CveContents = models.CveContents{}
+		}
+		v.CveContents[models.Microsoft] = *cveCont
+		r.ScannedCves[cveID] = v
+	}
+	return len(cveIDs), nil
+}
+
+// ConvertToModel converts gost model to vuls model
+func (ms Microsoft) ConvertToModel(cve *gostmodels.MicrosoftCVE) *models.CveContent {
+	v3score := 0.0
+	var v3Vector string
+	for _, scoreSet := range cve.ScoreSets {
+		if v3score < scoreSet.BaseScore {
+			v3score = scoreSet.BaseScore
+			v3Vector = scoreSet.Vector
+		}
+	}
+
+	var v3Severity string
+	for _, s := range cve.Severity {
+		v3Severity = s.Description
+	}
+
+	var refs []models.Reference
+	for _, r := range cve.References {
+		if r.AttrType == "External" {
+			refs = append(refs, models.Reference{Link: r.URL})
+		}
+	}
+
+	var cwe []string
+	if 0 < len(cve.CWE) {
+		cwe = []string{cve.CWE}
+	}
+
+	option := map[string]string{}
+	if 0 < len(cve.ExploitStatus) {
+		option["exploit"] = cve.ExploitStatus
+	}
+	if 0 < len(cve.Workaround) {
+		option["workaround"] = cve.Workaround
+	}
+	var kbids []string
+	for _, kbid := range cve.KBIDs {
+		kbids = append(kbids, kbid.KBID)
+	}
+	if 0 < len(kbids) {
+		option["kbids"] = strings.Join(kbids, ",")
+	}
+
+	return &models.CveContent{
+		Type:          models.Microsoft,
+		CveID:         cve.CveID,
+		Title:         cve.Title,
+		Summary:       cve.Description,
+		Cvss3Score:    v3score,
+		Cvss3Vector:   v3Vector,
+		Cvss3Severity: v3Severity,
+		References:    refs,
+		CweIDs:        cwe,
+		Mitigation:    cve.Mitigation,
+		Published:     cve.PublishDate,
+		LastModified:  cve.LastUpdateDate,
+		SourceLink:    "https://portal.msrc.microsoft.com/ja-jp/security-guidance/advisory/" + cve.CveID,
+		Optional:      option,
+	}
+}

gost/redhat.go

@@ -0,0 +1,289 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gost
+
+import (
+	"encoding/json"
+	"strconv"
+	"strings"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+	"github.com/knqyf263/gost/db"
+	gostmodels "github.com/knqyf263/gost/models"
+)
+
+// RedHat is Gost client for RedHat family linux
+type RedHat struct {
+	Base
+}
+
+// FillWithGost fills cve information that has in Gost
+func (red RedHat) FillWithGost(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
+	if nCVEs, err = red.fillUnfixed(driver, r); err != nil {
+		return 0, err
+	}
+	return nCVEs, red.fillFixed(driver, r)
+}
+
+func (red RedHat) fillFixed(driver db.DB, r *models.ScanResult) error {
+	var cveIDs []string
+	for cveID, vuln := range r.ScannedCves {
+		if _, ok := vuln.CveContents[models.RedHatAPI]; ok {
+			continue
+		}
+		cveIDs = append(cveIDs, cveID)
+	}
+
+	if config.Conf.Gost.IsFetchViaHTTP() {
+		prefix, _ := util.URLPathJoin(config.Conf.Gost.URL,
+			"redhat", "cves")
+		responses, err := getCvesViaHTTP(cveIDs, prefix)
+		if err != nil {
+			return err
+		}
+		for _, res := range responses {
+			redCve := gostmodels.RedhatCVE{}
+			if err := json.Unmarshal([]byte(res.json), &redCve); err != nil {
+				return err
+			}
+			if redCve.ID == 0 {
+				continue
+			}
+			cveCont := red.ConvertToModel(&redCve)
+			v, ok := r.ScannedCves[res.request.cveID]
+			if ok {
+				if v.CveContents == nil {
+					v.CveContents = models.NewCveContents(*cveCont)
+				} else {
+					v.CveContents[models.RedHatAPI] = *cveCont
+				}
+			} else {
+				v = models.VulnInfo{
+					CveID:       cveCont.CveID,
+					CveContents: models.NewCveContents(*cveCont),
+					Confidences: models.Confidences{models.RedHatAPIMatch},
+				}
+			}
+			r.ScannedCves[res.request.cveID] = v
+		}
+	} else {
+		if driver == nil {
+			return nil
+		}
+		for cveID, redCve := range driver.GetRedhatMulti(cveIDs) {
+			if redCve.ID == 0 {
+				continue
+			}
+			cveCont := red.ConvertToModel(&redCve)
+			v, ok := r.ScannedCves[cveID]
+			if ok {
+				if v.CveContents == nil {
+					v.CveContents = models.NewCveContents(*cveCont)
+				} else {
+					v.CveContents[models.RedHatAPI] = *cveCont
+				}
+			} else {
+				v = models.VulnInfo{
+					CveID:       cveCont.CveID,
+					CveContents: models.NewCveContents(*cveCont),
+					Confidences: models.Confidences{models.RedHatAPIMatch},
+				}
+			}
+			r.ScannedCves[cveID] = v
+		}
+	}
+
+	return nil
+}
+
+func (red RedHat) fillUnfixed(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
+	if config.Conf.Gost.IsFetchViaHTTP() {
+		prefix, _ := util.URLPathJoin(config.Conf.Gost.URL,
+			"redhat", major(r.Release), "pkgs")
+		responses, err := getAllUnfixedCvesViaHTTP(r, prefix)
+		if err != nil {
+			return 0, err
+		}
+		for _, res := range responses {
+			// CVE-ID: RedhatCVE
+			cves := map[string]gostmodels.RedhatCVE{}
+			if err := json.Unmarshal([]byte(res.json), &cves); err != nil {
+				return 0, err
+			}
+
+			for _, cve := range cves {
+				cveCont := red.ConvertToModel(&cve)
+				v, ok := r.ScannedCves[cve.Name]
+				if ok {
+					if v.CveContents == nil {
+						v.CveContents = models.NewCveContents(*cveCont)
+					} else {
+						v.CveContents[models.RedHatAPI] = *cveCont
+					}
+				} else {
+					v = models.VulnInfo{
+						CveID:       cveCont.CveID,
+						CveContents: models.NewCveContents(*cveCont),
+						Confidences: models.Confidences{models.RedHatAPIMatch},
+					}
+					nCVEs++
+				}
+				pkgStats := red.mergePackageStates(v,
+					cve.PackageState, r.Packages, r.Release)
+				if 0 < len(pkgStats) {
+					v.AffectedPackages = pkgStats
+					r.ScannedCves[cve.Name] = v
+				}
+			}
+		}
+	} else {
+		if driver == nil {
+			return 0, nil
+		}
+		for _, pack := range r.Packages {
+			// CVE-ID: RedhatCVE
+			cves := map[string]gostmodels.RedhatCVE{}
+			cves = driver.GetUnfixedCvesRedhat(major(r.Release), pack.Name)
+			for _, cve := range cves {
+				cveCont := red.ConvertToModel(&cve)
+				v, ok := r.ScannedCves[cve.Name]
+				if ok {
+					if v.CveContents == nil {
+						v.CveContents = models.NewCveContents(*cveCont)
+					} else {
+						v.CveContents[models.RedHatAPI] = *cveCont
+					}
+				} else {
+					v = models.VulnInfo{
+						CveID:       cveCont.CveID,
+						CveContents: models.NewCveContents(*cveCont),
+						Confidences: models.Confidences{models.RedHatAPIMatch},
+					}
+					nCVEs++
+				}
+
+				pkgStats := red.mergePackageStates(v,
+					cve.PackageState, r.Packages, r.Release)
+				if 0 < len(pkgStats) {
+					v.AffectedPackages = pkgStats
+					r.ScannedCves[cve.Name] = v
+				}
+			}
+		}
+	}
+	return nCVEs, nil
+}
+
+func (red RedHat) mergePackageStates(v models.VulnInfo, ps []gostmodels.RedhatPackageState, installed models.Packages, release string) (pkgStats models.PackageStatuses) {
+	pkgStats = v.AffectedPackages
+	for _, pstate := range ps {
+		if pstate.Cpe !=
+			"cpe:/o:redhat:enterprise_linux:"+major(release) {
+			return
+		}
+
+		if !(pstate.FixState == "Will not fix" ||
+			pstate.FixState == "Fix deferred") {
+			return
+		}
+
+		if _, ok := installed[pstate.PackageName]; !ok {
+			return
+		}
+
+		notFixedYet := false
+		switch pstate.FixState {
+		case "Will not fix", "Fix deferred":
+			notFixedYet = true
+		}
+
+		pkgStats = pkgStats.Store(models.PackageStatus{
+			Name:        pstate.PackageName,
+			FixState:    pstate.FixState,
+			NotFixedYet: notFixedYet,
+		})
+	}
+	return
+}
+
+func (red RedHat) parseCwe(str string) (cwes []string) {
+	if str != "" {
+		s := strings.Replace(str, "(", "|", -1)
+		s = strings.Replace(s, ")", "|", -1)
+		s = strings.Replace(s, "->", "|", -1)
+		for _, s := range strings.Split(s, "|") {
+			if s != "" {
+				cwes = append(cwes, s)
+			}
+		}
+	}
+	return
+}
+
+// ConvertToModel converts gost model to vuls model
+func (red RedHat) ConvertToModel(cve *gostmodels.RedhatCVE) *models.CveContent {
+	cwes := red.parseCwe(cve.Cwe)
+
+	details := []string{}
+	for _, detail := range cve.Details {
+		details = append(details, detail.Detail)
+	}
+
+	v2score := 0.0
+	if cve.Cvss.CvssBaseScore != "" {
+		v2score, _ = strconv.ParseFloat(cve.Cvss.CvssBaseScore, 64)
+	}
+	v2severity := ""
+	if v2score != 0 {
+		v2severity = cve.ThreatSeverity
+	}
+
+	v3score := 0.0
+	if cve.Cvss3.Cvss3BaseScore != "" {
+		v3score, _ = strconv.ParseFloat(cve.Cvss3.Cvss3BaseScore, 64)
+	}
+	v3severity := ""
+	if v3score != 0 {
+		v3severity = cve.ThreatSeverity
+	}
+
+	var refs []models.Reference
+	for _, r := range cve.References {
+		refs = append(refs, models.Reference{Link: r.Reference})
+	}
+
+	return &models.CveContent{
+		Type:          models.RedHatAPI,
+		CveID:         cve.Name,
+		Title:         cve.Bugzilla.Description,
+		Summary:       strings.Join(details, "\n"),
+		Cvss2Score:    v2score,
+		Cvss2Vector:   cve.Cvss.CvssScoringVector,
+		Cvss2Severity: v2severity,
+		Cvss3Score:    v3score,
+		Cvss3Vector:   cve.Cvss3.Cvss3ScoringVector,
+		Cvss3Severity: v3severity,
+		References:    refs,
+		CweIDs:        cwes,
+		Mitigation:    cve.Mitigation,
+		Published:     cve.PublicDate,
+		SourceLink:    "https://access.redhat.com/security/cve/" + cve.Name,
+	}
+}

gost/redhat_test.go

@@ -0,0 +1,37 @@
+package gost
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func TestParseCwe(t *testing.T) {
+	var tests = []struct {
+		in  string
+		out []string
+	}{
+		{
+			in:  "CWE-665->(CWE-200|CWE-89)",
+			out: []string{"CWE-665", "CWE-200", "CWE-89"},
+		},
+		{
+			in:  "CWE-841->CWE-770->CWE-454",
+			out: []string{"CWE-841", "CWE-770", "CWE-454"},
+		},
+		{
+			in:  "(CWE-122|CWE-125)",
+			out: []string{"CWE-122", "CWE-125"},
+		},
+	}
+
+	r := RedHat{}
+	for i, tt := range tests {
+		out := r.parseCwe(tt.in)
+		sort.Strings(out)
+		sort.Strings(tt.out)
+		if !reflect.DeepEqual(tt.out, out) {
+			t.Errorf("[%d]expected: %s, actual: %s", i, tt.out, out)
+		}
+	}
+}

gost/util.go

@@ -0,0 +1,201 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gost
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/cenkalti/backoff"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+	"github.com/parnurzeal/gorequest"
+)
+
+type response struct {
+	request request
+	json    string
+}
+
+func getCvesViaHTTP(cveIDs []string, urlPrefix string) (
+	responses []response, err error) {
+	nReq := len(cveIDs)
+	reqChan := make(chan request, nReq)
+	resChan := make(chan response, nReq)
+	errChan := make(chan error, nReq)
+	defer close(reqChan)
+	defer close(resChan)
+	defer close(errChan)
+
+	go func() {
+		for _, cveID := range cveIDs {
+			reqChan <- request{
+				cveID: cveID,
+			}
+		}
+	}()
+
+	concurrency := 10
+	tasks := util.GenWorkers(concurrency)
+	for i := 0; i < nReq; i++ {
+		tasks <- func() {
+			select {
+			case req := <-reqChan:
+				url, err := util.URLPathJoin(
+					urlPrefix,
+					req.cveID,
+				)
+				if err != nil {
+					errChan <- err
+				} else {
+					util.Log.Debugf("HTTP Request to %s", url)
+					httpGet(url, req, resChan, errChan)
+				}
+			}
+		}
+	}
+
+	timeout := time.After(2 * 60 * time.Second)
+	var errs []error
+	for i := 0; i < nReq; i++ {
+		select {
+		case res := <-resChan:
+			responses = append(responses, res)
+		case err := <-errChan:
+			errs = append(errs, err)
+		case <-timeout:
+			return nil, fmt.Errorf("Timeout Fetching OVAL")
+		}
+	}
+	if len(errs) != 0 {
+		return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
+	}
+	return
+}
+
+type request struct {
+	osMajorVersion string
+	packName       string
+	isSrcPack      bool
+	cveID          string
+}
+
+func getAllUnfixedCvesViaHTTP(r *models.ScanResult, urlPrefix string) (
+	responses []response, err error) {
+
+	nReq := len(r.Packages) + len(r.SrcPackages)
+	reqChan := make(chan request, nReq)
+	resChan := make(chan response, nReq)
+	errChan := make(chan error, nReq)
+	defer close(reqChan)
+	defer close(resChan)
+	defer close(errChan)
+
+	go func() {
+		for _, pack := range r.Packages {
+			reqChan <- request{
+				osMajorVersion: major(r.Release),
+				packName:       pack.Name,
+				isSrcPack:      false,
+			}
+		}
+		for _, pack := range r.SrcPackages {
+			reqChan <- request{
+				osMajorVersion: major(r.Release),
+				packName:       pack.Name,
+				isSrcPack:      true,
+			}
+		}
+	}()
+
+	concurrency := 10
+	tasks := util.GenWorkers(concurrency)
+	for i := 0; i < nReq; i++ {
+		tasks <- func() {
+			select {
+			case req := <-reqChan:
+				url, err := util.URLPathJoin(
+					urlPrefix,
+					req.packName,
+					"unfixed-cves",
+				)
+				if err != nil {
+					errChan <- err
+				} else {
+					util.Log.Debugf("HTTP Request to %s", url)
+					httpGet(url, req, resChan, errChan)
+				}
+			}
+		}
+	}
+
+	timeout := time.After(2 * 60 * time.Second)
+	var errs []error
+	for i := 0; i < nReq; i++ {
+		select {
+		case res := <-resChan:
+			responses = append(responses, res)
+		case err := <-errChan:
+			errs = append(errs, err)
+		case <-timeout:
+			return nil, fmt.Errorf("Timeout Fetching OVAL")
+		}
+	}
+	if len(errs) != 0 {
+		return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
+	}
+	return
+}
+
+func httpGet(url string, req request, resChan chan<- response, errChan chan<- error) {
+	var body string
+	var errs []error
+	var resp *http.Response
+	count, retryMax := 0, 3
+	f := func() (err error) {
+		//  resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
+		resp, body, errs = gorequest.New().Get(url).End()
+		if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
+			count++
+			if count == retryMax {
+				return nil
+			}
+			return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
+				errs, url, resp)
+		}
+		return nil
+	}
+	notify := func(err error, t time.Duration) {
+		util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
+	}
+	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
+	if err != nil {
+		errChan <- fmt.Errorf("HTTP Error %s", err)
+		return
+	}
+	if count == retryMax {
+		errChan <- fmt.Errorf("HRetry count exceeded")
+		return
+	}
+
+	resChan <- response{
+		request: req,
+		json:    body,
+	}
+}

main.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -25,15 +25,10 @@ import (
 	"context"
 
 	"github.com/future-architect/vuls/commands"
+	"github.com/future-architect/vuls/config"
 	"github.com/google/subcommands"
 )
 
-// Version of Vuls
-var version = "0.4.2"
-
-// Revision of Git
-var revision string
-
 func main() {
 	subcommands.Register(subcommands.HelpCommand(), "")
 	subcommands.Register(subcommands.FlagsCommand(), "")
@@ -44,13 +39,14 @@ func main() {
 	subcommands.Register(&commands.HistoryCmd{}, "history")
 	subcommands.Register(&commands.ReportCmd{}, "report")
 	subcommands.Register(&commands.ConfigtestCmd{}, "configtest")
+	subcommands.Register(&commands.ServerCmd{}, "server")
 
 	var v = flag.Bool("v", false, "Show version")
 
 	flag.Parse()
 
 	if *v {
-		fmt.Printf("vuls %s %s\n", version, revision)
+		fmt.Printf("vuls %s %s\n", config.Version, config.Revision)
 		os.Exit(int(subcommands.ExitSuccess))
 	}
 

models/cvecontents.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,7 +18,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package models
 
 import (
-	"strings"
 	"time"
 )
 
@@ -61,12 +60,12 @@ func (v CveContents) Except(exceptCtypes ...CveContentType) (values CveContents)
 // SourceLinks returns link of source
 func (v CveContents) SourceLinks(lang, myFamily, cveID string) (values []CveContentStr) {
 	if lang == "ja" {
-		if cont, found := v[JVN]; found && 0 < len(cont.SourceLink) {
-			values = append(values, CveContentStr{JVN, cont.SourceLink})
+		if cont, found := v[Jvn]; found && 0 < len(cont.SourceLink) {
+			values = append(values, CveContentStr{Jvn, cont.SourceLink})
 		}
 	}
 
-	order := CveContentTypes{NVD, NewCveContentType(myFamily)}
+	order := CveContentTypes{Nvd, NvdXML, NewCveContentType(myFamily)}
 	for _, ctype := range order {
 		if cont, found := v[ctype]; found {
 			values = append(values, CveContentStr{ctype, cont.SourceLink})
@@ -75,7 +74,7 @@ func (v CveContents) SourceLinks(lang, myFamily, cveID string) (values []CveCont
 
 	if len(values) == 0 {
 		return []CveContentStr{{
-			Type:  NVD,
+			Type:  Nvd,
 			Value: "https://nvd.nist.gov/vuln/detail/" + cveID,
 		}}
 	}
@@ -148,11 +147,14 @@ func (v CveContents) References(myFamily string) (values []CveContentRefs) {
 func (v CveContents) CweIDs(myFamily string) (values []CveContentStr) {
 	order := CveContentTypes{NewCveContentType(myFamily)}
 	order = append(order, AllCveContetTypes.Except(append(order)...)...)
-
 	for _, ctype := range order {
-		if cont, found := v[ctype]; found && 0 < len(cont.CweID) {
-			// RedHat's OVAL sometimes contains multiple CWE-IDs separated by spaces
-			for _, cweID := range strings.Fields(cont.CweID) {
+		if cont, found := v[ctype]; found && 0 < len(cont.CweIDs) {
+			for _, cweID := range cont.CweIDs {
+				for _, val := range values {
+					if val.Value == cweID {
+						continue
+					}
+				}
 				values = append(values, CveContentStr{
 					Type:  ctype,
 					Value: cweID,
@@ -163,23 +165,38 @@ func (v CveContents) CweIDs(myFamily string) (values []CveContentStr) {
 	return
 }
 
+// UniqCweIDs returns Uniq CweIDs
+func (v CveContents) UniqCweIDs(myFamily string) (values []CveContentStr) {
+	uniq := map[string]CveContentStr{}
+	for _, cwes := range v.CweIDs(myFamily) {
+		uniq[cwes.Value] = cwes
+	}
+	for _, cwe := range uniq {
+		values = append(values, cwe)
+	}
+	return values
+}
+
 // CveContent has abstraction of various vulnerability information
 type CveContent struct {
-	Type         CveContentType
-	CveID        string
-	Title        string
-	Summary      string
-	Severity     string
-	Cvss2Score   float64
-	Cvss2Vector  string
-	Cvss3Score   float64
-	Cvss3Vector  string
-	SourceLink   string
-	Cpes         []Cpe
-	References   References
-	CweID        string
-	Published    time.Time
-	LastModified time.Time
+	Type          CveContentType    `json:"type"`
+	CveID         string            `json:"cveID"`
+	Title         string            `json:"title"`
+	Summary       string            `json:"summary"`
+	Cvss2Score    float64           `json:"cvss2Score"`
+	Cvss2Vector   string            `json:"cvss2Vector"`
+	Cvss2Severity string            `json:"cvss2Severity"`
+	Cvss3Score    float64           `json:"cvss3Score"`
+	Cvss3Vector   string            `json:"cvss3Vector"`
+	Cvss3Severity string            `json:"cvss3Severity"`
+	SourceLink    string            `json:"sourceLink"`
+	Cpes          []Cpe             `json:"cpes,omitempty"`
+	References    References        `json:"references,omitempty"`
+	CweIDs        []string          `json:"cweIDs,omitempty"`
+	Published     time.Time         `json:"published"`
+	LastModified  time.Time         `json:"lastModified"`
+	Mitigation    string            `json:"mitigation"` // RedHat API
+	Optional      map[string]string `json:"optional,omitempty"`
 }
 
 // Empty checks the content is empty
@@ -193,10 +210,12 @@ type CveContentType string
 // NewCveContentType create CveContentType
 func NewCveContentType(name string) CveContentType {
 	switch name {
+	case "nvdxml":
+		return NvdXML
 	case "nvd":
-		return NVD
+		return Nvd
 	case "jvn":
-		return JVN
+		return Jvn
 	case "redhat", "centos":
 		return RedHat
 	case "oracle":
@@ -205,21 +224,36 @@ func NewCveContentType(name string) CveContentType {
 		return Ubuntu
 	case "debian":
 		return Debian
+	case "redhat_api":
+		return RedHatAPI
+	case "debian_security_tracker":
+		return DebianSecurityTracker
+	case "microsoft":
+		return Microsoft
 	default:
 		return Unknown
 	}
 }
 
 const (
-	// NVD is NVD
-	NVD CveContentType = "nvd"
+	// NvdXML is NvdXML
+	NvdXML CveContentType = "nvdxml"
 
-	// JVN is JVN
-	JVN CveContentType = "jvn"
+	// Nvd is Nvd
+	Nvd CveContentType = "nvd"
+
+	// Jvn is Jvn
+	Jvn CveContentType = "jvn"
 
 	// RedHat is RedHat
 	RedHat CveContentType = "redhat"
 
+	// RedHatAPI is RedHat
+	RedHatAPI CveContentType = "redhat_api"
+
+	// DebianSecurityTracker is Debian Secury tracker
+	DebianSecurityTracker CveContentType = "debian_security_tracker"
+
 	// Debian is Debian
 	Debian CveContentType = "debian"
 
@@ -232,6 +266,9 @@ const (
 	// SUSE is SUSE Linux
 	SUSE CveContentType = "suse"
 
+	// Microsoft is Microsoft
+	Microsoft CveContentType = "microsoft"
+
 	// Unknown is Unknown
 	Unknown CveContentType = "unknown"
 )
@@ -240,7 +277,16 @@ const (
 type CveContentTypes []CveContentType
 
 // AllCveContetTypes has all of CveContentTypes
-var AllCveContetTypes = CveContentTypes{NVD, JVN, RedHat, Debian, Ubuntu}
+var AllCveContetTypes = CveContentTypes{
+	Nvd,
+	NvdXML,
+	Jvn,
+	RedHat,
+	Debian,
+	Ubuntu,
+	RedHatAPI,
+	DebianSecurityTracker,
+}
 
 // Except returns CveContentTypes except for given args
 func (c CveContentTypes) Except(excepts ...CveContentType) (excepted CveContentTypes) {
@@ -261,7 +307,8 @@ func (c CveContentTypes) Except(excepts ...CveContentType) (excepted CveContentT
 
 // Cpe is Common Platform Enumeration
 type Cpe struct {
-	CpeName string
+	URI             string `json:"uri"`
+	FormattedString string `json:"formattedString"`
 }
 
 // References is a slice of Reference
@@ -269,7 +316,7 @@ type References []Reference
 
 // Reference has a related link of the CVE
 type Reference struct {
-	Source string
-	Link   string
-	RefID  string
+	Source string `json:"source"`
+	Link   string `json:"link"`
+	RefID  string `json:"refID"`
 }

models/cvecontents_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -60,27 +60,27 @@ func TestSourceLinks(t *testing.T) {
 				lang:  "ja",
 				cveID: "CVE-2017-6074",
 				cont: CveContents{
-					JVN: {
-						Type:       JVN,
+					Jvn: {
+						Type:       Jvn,
 						SourceLink: "https://jvn.jp/vu/JVNVU93610402/",
 					},
 					RedHat: {
 						Type:       RedHat,
 						SourceLink: "https://access.redhat.com/security/cve/CVE-2017-6074",
 					},
-					NVD: {
-						Type:       NVD,
+					NvdXML: {
+						Type:       NvdXML,
 						SourceLink: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074",
 					},
 				},
 			},
 			out: []CveContentStr{
 				{
-					Type:  JVN,
+					Type:  Jvn,
 					Value: "https://jvn.jp/vu/JVNVU93610402/",
 				},
 				{
-					Type:  NVD,
+					Type:  NvdXML,
 					Value: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074",
 				},
 				{
@@ -95,23 +95,23 @@ func TestSourceLinks(t *testing.T) {
 				lang:  "en",
 				cveID: "CVE-2017-6074",
 				cont: CveContents{
-					JVN: {
-						Type:       JVN,
+					Jvn: {
+						Type:       Jvn,
 						SourceLink: "https://jvn.jp/vu/JVNVU93610402/",
 					},
 					RedHat: {
 						Type:       RedHat,
 						SourceLink: "https://access.redhat.com/security/cve/CVE-2017-6074",
 					},
-					NVD: {
-						Type:       NVD,
+					NvdXML: {
+						Type:       NvdXML,
 						SourceLink: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074",
 					},
 				},
 			},
 			out: []CveContentStr{
 				{
-					Type:  NVD,
+					Type:  NvdXML,
 					Value: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074",
 				},
 				{
@@ -129,16 +129,16 @@ func TestSourceLinks(t *testing.T) {
 			},
 			out: []CveContentStr{
 				{
-					Type:  NVD,
+					Type:  Nvd,
 					Value: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074",
 				},
 			},
 		},
 	}
-	for _, tt := range tests {
+	for i, tt := range tests {
 		actual := tt.in.cont.SourceLinks(tt.in.lang, "redhat", tt.in.cveID)
 		if !reflect.DeepEqual(tt.out, actual) {
-			t.Errorf("\nexpected: %v\n  actual: %v\n", tt.out, actual)
+			t.Errorf("\n[%d] expected: %v\n  actual: %v\n", i, tt.out, actual)
 		}
 	}
 }
@@ -158,16 +158,16 @@ func TestVendorLink(t *testing.T) {
 				vinfo: VulnInfo{
 					CveID: "CVE-2017-6074",
 					CveContents: CveContents{
-						JVN: {
-							Type:       JVN,
+						Jvn: {
+							Type:       Jvn,
 							SourceLink: "https://jvn.jp/vu/JVNVU93610402/",
 						},
 						RedHat: {
 							Type:       RedHat,
 							SourceLink: "https://access.redhat.com/security/cve/CVE-2017-6074",
 						},
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							SourceLink: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074",
 						},
 					},

models/models.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,4 +18,4 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package models
 
 // JSONVersion is JSON Version
-const JSONVersion = 3
+const JSONVersion = 4

models/models_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

models/packages.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -38,6 +38,12 @@ func NewPackages(packs ...Package) Packages {
 
 // MergeNewVersion merges candidate version information to the receiver struct
 func (ps Packages) MergeNewVersion(as Packages) {
+	for name, pack := range ps {
+		pack.NewVersion = pack.Version
+		pack.NewRelease = pack.Release
+		ps[name] = pack
+	}
+
 	for _, a := range as {
 		if pack, ok := ps[a.Name]; ok {
 			pack.NewVersion = a.NewVersion
@@ -60,18 +66,7 @@ func (ps Packages) Merge(other Packages) Packages {
 	return merged
 }
 
-// FormatUpdatablePacksSummary returns a summary of updatable packages
-func (ps Packages) FormatUpdatablePacksSummary() string {
-	nUpdatable := 0
-	for _, p := range ps {
-		if p.NewVersion != "" {
-			nUpdatable++
-		}
-	}
-	return fmt.Sprintf("%d updatable packages", nUpdatable)
-}
-
-// FindOne search a element by name-newver-newrel-arch
+// FindOne search a element
 func (ps Packages) FindOne(f func(Package) bool) (string, Package, bool) {
 	for key, p := range ps {
 		if f(p) {
@@ -81,16 +76,44 @@ func (ps Packages) FindOne(f func(Package) bool) (string, Package, bool) {
 	return "", Package{}, false
 }
 
+// FindByFQPN search a package by Fully-Qualified-Package-Name
+func (ps Packages) FindByFQPN(nameVerRelArc string) (*Package, error) {
+	for _, p := range ps {
+		if nameVerRelArc == p.FQPN() {
+			return &p, nil
+		}
+	}
+	return nil, fmt.Errorf("Failed to find the package: %s", nameVerRelArc)
+}
+
 // Package has installed binary packages.
 type Package struct {
-	Name       string
-	Version    string
-	Release    string
-	NewVersion string
-	NewRelease string
-	Arch       string
-	Repository string
-	Changelog  Changelog
+	Name             string               `json:"name"`
+	Version          string               `json:"version"`
+	Release          string               `json:"release"`
+	NewVersion       string               `json:"newVersion"`
+	NewRelease       string               `json:"newRelease"`
+	Arch             string               `json:"arch"`
+	Repository       string               `json:"repository"`
+	Changelog        Changelog            `json:"changelog"`
+	AffectedProcs    []AffectedProcess    `json:",omitempty"`
+	NeedRestartProcs []NeedRestartProcess `json:",omitempty"`
+}
+
+// FQPN returns Fully-Qualified-Package-Name
+// name-version-release.arch
+func (p Package) FQPN() string {
+	fqpn := p.Name
+	if p.Version != "" {
+		fqpn += fmt.Sprintf("-%s", p.Version)
+	}
+	if p.Release != "" {
+		fqpn += fmt.Sprintf("-%s", p.Release)
+	}
+	if p.Arch != "" {
+		fqpn += fmt.Sprintf(".%s", p.Arch)
+	}
+	return fqpn
 }
 
 // FormatVer returns package version-release
@@ -112,10 +135,14 @@ func (p Package) FormatNewVer() string {
 }
 
 // FormatVersionFromTo formats installed and new package version
-func (p Package) FormatVersionFromTo(notFixedYet bool) string {
+func (p Package) FormatVersionFromTo(notFixedYet bool, status string) string {
 	to := p.FormatNewVer()
 	if notFixedYet {
-		to = "Not Fixed Yet"
+		if status != "" {
+			to = status
+		} else {
+			to = "Not Fixed Yet"
+		}
 	} else if p.NewVersion == "" {
 		to = "Unknown"
 	}
@@ -141,7 +168,7 @@ func (p Package) FormatChangelog() string {
 	case FailedToGetChangelog:
 		clog = "No changelogs"
 	case FailedToFindVersionInChangelog:
-		clog = "Failed to parse changelogs. For detials, check yourself"
+		clog = "Failed to parse changelogs. For details, check yourself"
 	}
 	buf = append(buf, packVer, delim.String(), clog)
 	return strings.Join(buf, "\n")
@@ -150,8 +177,23 @@ func (p Package) FormatChangelog() string {
 // Changelog has contents of changelog and how to get it.
 // Method: models.detectionMethodStr
 type Changelog struct {
-	Contents string
-	Method   DetectionMethod
+	Contents string          `json:"contents"`
+	Method   DetectionMethod `json:"method"`
+}
+
+// AffectedProcess keep a processes information affected by software update
+type AffectedProcess struct {
+	PID  string `json:"pid"`
+	Name string `json:"name"`
+}
+
+// NeedRestartProcess keep a processes information affected by software update
+type NeedRestartProcess struct {
+	PID         string `json:"pid"`
+	Path        string `json:"path"`
+	ServiceName string `json:"serviceName"`
+	InitSystem  string `json:"initSystem"`
+	HasInit     bool   `json:"-"`
 }
 
 // SrcPackage has installed source package information.
@@ -160,9 +202,9 @@ type Changelog struct {
 // so it is also needed to capture source version for OVAL version comparison.
 // https://github.com/future-architect/vuls/issues/504
 type SrcPackage struct {
-	Name        string
-	Version     string
-	BinaryNames []string
+	Name        string   `json:"name"`
+	Version     string   `json:"version"`
+	BinaryNames []string `json:"binaryNames"`
 }
 
 // AddBinaryName add the name if not exists
@@ -181,3 +223,15 @@ func (s *SrcPackage) AddBinaryName(name string) {
 // SrcPackages is Map of SrcPackage
 // { "package-name": SrcPackage }
 type SrcPackages map[string]SrcPackage
+
+// FindByBinName finds by bin-package-name
+func (s SrcPackages) FindByBinName(name string) (*SrcPackage, bool) {
+	for _, p := range s {
+		for _, binName := range p.BinaryNames {
+			if binName == name {
+				return &p, true
+			}
+		}
+	}
+	return nil, false
+}

models/packages_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -133,3 +133,61 @@ func TestAddBinaryName(t *testing.T) {
 		}
 	}
 }
+
+func TestFindByBinName(t *testing.T) {
+	var tests = []struct {
+		in       SrcPackages
+		name     string
+		expected *SrcPackage
+		ok       bool
+	}{
+		{
+			in: map[string]SrcPackage{
+				"packA": {
+					Name:        "srcA",
+					BinaryNames: []string{"binA"},
+					Version:     "1.0.0",
+				},
+				"packB": {
+					Name:        "srcB",
+					BinaryNames: []string{"binB"},
+					Version:     "2.0.0",
+				},
+			},
+			name: "binA",
+			expected: &SrcPackage{
+				Name:        "srcA",
+				BinaryNames: []string{"binA"},
+				Version:     "1.0.0",
+			},
+			ok: true,
+		},
+		{
+			in: map[string]SrcPackage{
+				"packA": {
+					Name:        "srcA",
+					BinaryNames: []string{"binA"},
+					Version:     "1.0.0",
+				},
+				"packB": {
+					Name:        "srcB",
+					BinaryNames: []string{"binB"},
+					Version:     "2.0.0",
+				},
+			},
+			name:     "nobin",
+			expected: nil,
+			ok:       false,
+		},
+	}
+
+	for i, tt := range tests {
+		act, ok := tt.in.FindByBinName(tt.name)
+		if ok != tt.ok {
+			t.Errorf("[%d] expected %#v, actual %#v", i, tt.in, tt.expected)
+		}
+		if act != nil && !reflect.DeepEqual(*tt.expected, *act) {
+			t.Errorf("[%d] expected %#v, actual %#v", i, tt.in, tt.expected)
+		}
+	}
+}

models/scanresults.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,9 +20,13 @@ package models
 import (
 	"bytes"
 	"fmt"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/cwe"
+	"github.com/future-architect/vuls/util"
 )
 
 // ScanResults is a slide of ScanResult
@@ -30,38 +34,84 @@ type ScanResults []ScanResult
 
 // ScanResult has the result of scanned CVE information.
 type ScanResult struct {
-	ScannedAt   time.Time
-	ReportedAt  time.Time
-	JSONVersion int
-	Lang        string
-	ServerUUID  string
-	ServerName  string // TOML Section key
-	Family      string
-	Release     string
-	Container   Container
-	Platform    Platform
+	JSONVersion      int                    `json:"jsonVersion"`
+	Lang             string                 `json:"lang"`
+	ServerUUID       string                 `json:"serverUUID"`
+	ServerName       string                 `json:"serverName"` // TOML Section key
+	Family           string                 `json:"family"`
+	Release          string                 `json:"release"`
+	Container        Container              `json:"container"`
+	Platform         Platform               `json:"platform"`
+	IPv4Addrs        []string               `json:"ipv4Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast)
+	IPv6Addrs        []string               `json:"ipv6Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast)
+	ScannedAt        time.Time              `json:"scannedAt"`
+	ScanMode         string                 `json:"scanMode"`
+	ScannedVersion   string                 `json:"scannedVersion"`
+	ScannedRevision  string                 `json:"scannedRevision"`
+	ScannedBy        string                 `json:"scannedBy"`
+	ReportedAt       time.Time              `json:"reportedAt"`
+	ReportedVersion  string                 `json:"reportedVersion"`
+	ReportedRevision string                 `json:"reportedRevision"`
+	ReportedBy       string                 `json:"reportedBy"`
+	ScannedCves      VulnInfos              `json:"scannedCves"`
+	RunningKernel    Kernel                 `json:"runningKernel"`
+	Packages         Packages               `json:"packages"`
+	CweDict          CweDict                `json:"cweDict"`
+	Optional         map[string]interface{} `json:",omitempty"`
+	SrcPackages      SrcPackages            `json:",omitempty"`
+	Errors           []string               `json:"errors"`
+	Config           struct {
+		Scan   config.Config `json:"scan"`
+		Report config.Config `json:"report"`
+	} `json:"config"`
+}
 
-	// Scanned Vulns by SSH scan + CPE + OVAL
-	ScannedCves VulnInfos
+// CweDict is a dictionary for CWE
+type CweDict map[string]CweDictEntry
 
-	RunningKernel Kernel
-	Packages      Packages
-	SrcPackages   SrcPackages
-
-	Errors   []string
-	Optional [][]interface{}
-
-	Config struct {
-		Scan   config.Config
-		Report config.Config
+// Get the name, url, top10URL for the specified cweID, lang
+func (c CweDict) Get(cweID, lang string) (name, url, top10Rank, top10URL string) {
+	cweNum := strings.TrimPrefix(cweID, "CWE-")
+	switch config.Conf.Lang {
+	case "ja":
+		if dict, ok := c[cweNum]; ok && dict.OwaspTopTen2017 != "" {
+			top10Rank = dict.OwaspTopTen2017
+			top10URL = cwe.OwaspTopTen2017GitHubURLJa[dict.OwaspTopTen2017]
+		}
+		if dict, ok := cwe.CweDictJa[cweNum]; ok {
+			name = dict.Name
+			url = fmt.Sprintf("http://jvndb.jvn.jp/ja/cwe/%s.html", cweID)
+		} else {
+			if dict, ok := cwe.CweDictEn[cweNum]; ok {
+				name = dict.Name
+			}
+			url = fmt.Sprintf("https://cwe.mitre.org/data/definitions/%s.html", cweID)
+		}
+	default:
+		if dict, ok := c[cweNum]; ok && dict.OwaspTopTen2017 != "" {
+			top10Rank = dict.OwaspTopTen2017
+			top10URL = cwe.OwaspTopTen2017GitHubURLEn[dict.OwaspTopTen2017]
+		}
+		url = fmt.Sprintf("https://cwe.mitre.org/data/definitions/%s.html", cweID)
+		if dict, ok := cwe.CweDictEn[cweNum]; ok {
+			name = dict.Name
+		}
 	}
+	return
+}
+
+// CweDictEntry is a entry of CWE
+type CweDictEntry struct {
+	En              *cwe.Cwe `json:"en,omitempty"`
+	Ja              *cwe.Cwe `json:"ja,omitempty"`
+	OwaspTopTen2017 string   `json:"owaspTopTen2017"`
 }
 
 // Kernel has the Release, version and whether need restart
 type Kernel struct {
-	Release        string
-	Version        string
-	RebootRequired bool
+	Release        string `json:"release"`
+	Version        string `json:"version"`
+	RebootRequired bool   `json:"rebootRequired"`
 }
 
 // FilterByCvssOver is filter function.
@@ -83,9 +133,29 @@ func (r ScanResult) FilterByCvssOver(over float64) ScanResult {
 }
 
 // FilterIgnoreCves is filter function.
-func (r ScanResult) FilterIgnoreCves(cveIDs []string) ScanResult {
+func (r ScanResult) FilterIgnoreCves() ScanResult {
+
+	ignoreCves := []string{}
+	if len(r.Container.Name) == 0 {
+		ignoreCves = config.Conf.Servers[r.ServerName].IgnoreCves
+	} else {
+		if s, ok := config.Conf.Servers[r.ServerName]; ok {
+			if con, ok := s.Containers[r.Container.Name]; ok {
+				ignoreCves = con.IgnoreCves
+			} else {
+				util.Log.Errorf("%s is not found in config.toml",
+					r.Container.Name)
+				return r
+			}
+		} else {
+			util.Log.Errorf("%s is not found in config.toml",
+				r.ServerName)
+			return r
+		}
+	}
+
 	filtered := r.ScannedCves.Find(func(v VulnInfo) bool {
-		for _, c := range cveIDs {
+		for _, c := range ignoreCves {
 			if v.CveID == c {
 				return false
 			}
@@ -112,6 +182,63 @@ func (r ScanResult) FilterUnfixed() ScanResult {
 	return r
 }
 
+// FilterIgnorePkgs is filter function.
+func (r ScanResult) FilterIgnorePkgs() ScanResult {
+	ignorePkgsRegexps := []string{}
+	if len(r.Container.Name) == 0 {
+		ignorePkgsRegexps = config.Conf.Servers[r.ServerName].IgnorePkgsRegexp
+	} else {
+		if s, ok := config.Conf.Servers[r.ServerName]; ok {
+			if con, ok := s.Containers[r.Container.Name]; ok {
+				ignorePkgsRegexps = con.IgnorePkgsRegexp
+			} else {
+				util.Log.Errorf("%s is not found in config.toml",
+					r.Container.Name)
+				return r
+			}
+		} else {
+			util.Log.Errorf("%s is not found in config.toml",
+				r.ServerName)
+			return r
+		}
+	}
+
+	regexps := []*regexp.Regexp{}
+	for _, pkgRegexp := range ignorePkgsRegexps {
+		re, err := regexp.Compile(pkgRegexp)
+		if err != nil {
+			util.Log.Errorf("Faild to parse %s, %s", pkgRegexp, err)
+			continue
+		} else {
+			regexps = append(regexps, re)
+		}
+	}
+	if len(regexps) == 0 {
+		return r
+	}
+
+	filtered := r.ScannedCves.Find(func(v VulnInfo) bool {
+		if len(v.AffectedPackages) == 0 {
+			return true
+		}
+		for _, p := range v.AffectedPackages {
+			match := false
+			for _, re := range regexps {
+				if re.MatchString(p.Name) {
+					match = true
+				}
+			}
+			if !match {
+				return true
+			}
+		}
+		return false
+	})
+
+	r.ScannedCves = filtered
+	return r
+}
+
 // ReportFileName returns the filename on localhost without extention
 func (r ScanResult) ReportFileName() (name string) {
 	if len(r.Container.ContainerID) == 0 {
@@ -144,7 +271,7 @@ func (r ScanResult) ServerInfo() string {
 	)
 }
 
-// ServerInfoTui returns server infromation for TUI sidebar
+// ServerInfoTui returns server information for TUI sidebar
 func (r ScanResult) ServerInfoTui() string {
 	if len(r.Container.ContainerID) == 0 {
 		line := fmt.Sprintf("%s (%s%s)",
@@ -178,29 +305,106 @@ func (r ScanResult) FormatServerName() (name string) {
 
 // FormatTextReportHeadedr returns header of text report
 func (r ScanResult) FormatTextReportHeadedr() string {
-	serverInfo := r.ServerInfo()
 	var buf bytes.Buffer
-	for i := 0; i < len(serverInfo); i++ {
+	for i := 0; i < len(r.ServerInfo()); i++ {
 		buf.WriteString("=")
 	}
-	return fmt.Sprintf("%s\n%s\n%s\t%s\n",
+
+	return fmt.Sprintf("%s\n%s\n%s, %s, %s, %s\n",
 		r.ServerInfo(),
 		buf.String(),
 		r.ScannedCves.FormatCveSummary(),
-		r.Packages.FormatUpdatablePacksSummary(),
+		r.ScannedCves.FormatFixedStatus(r.Packages),
+		r.FormatUpdatablePacksSummary(),
+		r.FormatExploitCveSummary(),
 	)
 }
 
+// FormatUpdatablePacksSummary returns a summary of updatable packages
+func (r ScanResult) FormatUpdatablePacksSummary() string {
+	if !r.isDisplayUpdatableNum() {
+		return fmt.Sprintf("%d installed", len(r.Packages))
+	}
+
+	nUpdatable := 0
+	for _, p := range r.Packages {
+		if p.NewVersion == "" {
+			continue
+		}
+		if p.Version != p.NewVersion || p.Release != p.NewRelease {
+			nUpdatable++
+		}
+	}
+	return fmt.Sprintf("%d installed, %d updatable",
+		len(r.Packages),
+		nUpdatable)
+}
+
+// FormatExploitCveSummary returns a summary of exploit cve
+func (r ScanResult) FormatExploitCveSummary() string {
+	nExploitCve := 0
+	for _, vuln := range r.ScannedCves {
+		if 0 < len(vuln.Exploits) {
+			nExploitCve++
+		}
+	}
+	return fmt.Sprintf("%d exploits", nExploitCve)
+}
+
+func (r ScanResult) isDisplayUpdatableNum() bool {
+	var mode config.ScanMode
+	s, _ := config.Conf.Servers[r.ServerName]
+	mode = s.Mode
+
+	if mode.IsOffline() {
+		return false
+	}
+	if mode.IsFastRoot() || mode.IsDeep() {
+		return true
+	}
+	if mode.IsFast() {
+		switch r.Family {
+		case config.RedHat,
+			config.Oracle,
+			config.Debian,
+			config.Ubuntu,
+			config.Raspbian:
+			return false
+		default:
+			return true
+		}
+	}
+	return false
+}
+
+// IsContainer returns whether this ServerInfo is about container
+func (r ScanResult) IsContainer() bool {
+	return 0 < len(r.Container.ContainerID)
+}
+
+// IsDeepScanMode checks if the scan mode is deep scan mode.
+func (r ScanResult) IsDeepScanMode() bool {
+	for _, s := range r.Config.Scan.Servers {
+		for _, m := range s.ScanMode {
+			if m == "deep" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // Container has Container information
 type Container struct {
-	ContainerID string
-	Name        string
-	Image       string
-	Type        string
+	ContainerID string `json:"containerID"`
+	Name        string `json:"name"`
+	Image       string `json:"image"`
+	Type        string `json:"type"`
+	UUID        string `json:"uuid"`
 }
 
 // Platform has platform information
 type Platform struct {
-	Name       string // aws or azure or gcp or other...
-	InstanceID string
+	Name       string `json:"name"` // aws or azure or gcp or other...
+	InstanceID string `json:"instanceID"`
 }

models/scanresults_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -43,7 +43,7 @@ func TestFilterByCvssOver(t *testing.T) {
 							CveID: "CVE-2017-0001",
 							CveContents: NewCveContents(
 								CveContent{
-									Type:         NVD,
+									Type:         NvdXML,
 									CveID:        "CVE-2017-0001",
 									Cvss2Score:   7.1,
 									LastModified: time.Time{},
@@ -54,7 +54,7 @@ func TestFilterByCvssOver(t *testing.T) {
 							CveID: "CVE-2017-0002",
 							CveContents: NewCveContents(
 								CveContent{
-									Type:         NVD,
+									Type:         NvdXML,
 									CveID:        "CVE-2017-0002",
 									Cvss2Score:   6.9,
 									LastModified: time.Time{},
@@ -65,13 +65,13 @@ func TestFilterByCvssOver(t *testing.T) {
 							CveID: "CVE-2017-0003",
 							CveContents: NewCveContents(
 								CveContent{
-									Type:         NVD,
+									Type:         NvdXML,
 									CveID:        "CVE-2017-0003",
 									Cvss2Score:   6.9,
 									LastModified: time.Time{},
 								},
 								CveContent{
-									Type:         JVN,
+									Type:         Jvn,
 									CveID:        "CVE-2017-0003",
 									Cvss2Score:   7.2,
 									LastModified: time.Time{},
@@ -87,7 +87,7 @@ func TestFilterByCvssOver(t *testing.T) {
 						CveID: "CVE-2017-0001",
 						CveContents: NewCveContents(
 							CveContent{
-								Type:         NVD,
+								Type:         NvdXML,
 								CveID:        "CVE-2017-0001",
 								Cvss2Score:   7.1,
 								LastModified: time.Time{},
@@ -98,13 +98,13 @@ func TestFilterByCvssOver(t *testing.T) {
 						CveID: "CVE-2017-0003",
 						CveContents: NewCveContents(
 							CveContent{
-								Type:         NVD,
+								Type:         NvdXML,
 								CveID:        "CVE-2017-0003",
 								Cvss2Score:   6.9,
 								LastModified: time.Time{},
 							},
 							CveContent{
-								Type:         JVN,
+								Type:         Jvn,
 								CveID:        "CVE-2017-0003",
 								Cvss2Score:   7.2,
 								LastModified: time.Time{},
@@ -124,10 +124,10 @@ func TestFilterByCvssOver(t *testing.T) {
 							CveID: "CVE-2017-0001",
 							CveContents: NewCveContents(
 								CveContent{
-									Type:         Ubuntu,
-									CveID:        "CVE-2017-0001",
-									Severity:     "HIGH",
-									LastModified: time.Time{},
+									Type:          Ubuntu,
+									CveID:         "CVE-2017-0001",
+									Cvss2Severity: "HIGH",
+									LastModified:  time.Time{},
 								},
 							),
 						},
@@ -135,10 +135,10 @@ func TestFilterByCvssOver(t *testing.T) {
 							CveID: "CVE-2017-0002",
 							CveContents: NewCveContents(
 								CveContent{
-									Type:         RedHat,
-									CveID:        "CVE-2017-0002",
-									Severity:     "CRITICAL",
-									LastModified: time.Time{},
+									Type:          RedHat,
+									CveID:         "CVE-2017-0002",
+									Cvss2Severity: "CRITICAL",
+									LastModified:  time.Time{},
 								},
 							),
 						},
@@ -146,10 +146,10 @@ func TestFilterByCvssOver(t *testing.T) {
 							CveID: "CVE-2017-0003",
 							CveContents: NewCveContents(
 								CveContent{
-									Type:         Oracle,
-									CveID:        "CVE-2017-0003",
-									Severity:     "IMPORTANT",
-									LastModified: time.Time{},
+									Type:          Oracle,
+									CveID:         "CVE-2017-0003",
+									Cvss2Severity: "IMPORTANT",
+									LastModified:  time.Time{},
 								},
 							),
 						},
@@ -162,10 +162,10 @@ func TestFilterByCvssOver(t *testing.T) {
 						CveID: "CVE-2017-0001",
 						CveContents: NewCveContents(
 							CveContent{
-								Type:         Ubuntu,
-								CveID:        "CVE-2017-0001",
-								Severity:     "HIGH",
-								LastModified: time.Time{},
+								Type:          Ubuntu,
+								CveID:         "CVE-2017-0001",
+								Cvss2Severity: "HIGH",
+								LastModified:  time.Time{},
 							},
 						),
 					},
@@ -173,10 +173,10 @@ func TestFilterByCvssOver(t *testing.T) {
 						CveID: "CVE-2017-0002",
 						CveContents: NewCveContents(
 							CveContent{
-								Type:         RedHat,
-								CveID:        "CVE-2017-0002",
-								Severity:     "CRITICAL",
-								LastModified: time.Time{},
+								Type:          RedHat,
+								CveID:         "CVE-2017-0002",
+								Cvss2Severity: "CRITICAL",
+								LastModified:  time.Time{},
 							},
 						),
 					},
@@ -184,10 +184,10 @@ func TestFilterByCvssOver(t *testing.T) {
 						CveID: "CVE-2017-0003",
 						CveContents: NewCveContents(
 							CveContent{
-								Type:         Oracle,
-								CveID:        "CVE-2017-0003",
-								Severity:     "IMPORTANT",
-								LastModified: time.Time{},
+								Type:          Oracle,
+								CveID:         "CVE-2017-0003",
+								Cvss2Severity: "IMPORTANT",
+								LastModified:  time.Time{},
 							},
 						),
 					},
@@ -206,7 +206,6 @@ func TestFilterByCvssOver(t *testing.T) {
 		}
 	}
 }
-
 func TestFilterIgnoreCveIDs(t *testing.T) {
 	type in struct {
 		cves []string
@@ -220,6 +219,7 @@ func TestFilterIgnoreCveIDs(t *testing.T) {
 			in: in{
 				cves: []string{"CVE-2017-0002"},
 				rs: ScanResult{
+					ServerName: "name",
 					ScannedCves: VulnInfos{
 						"CVE-2017-0001": {
 							CveID: "CVE-2017-0001",
@@ -234,6 +234,7 @@ func TestFilterIgnoreCveIDs(t *testing.T) {
 				},
 			},
 			out: ScanResult{
+				ServerName: "name",
 				ScannedCves: VulnInfos{
 					"CVE-2017-0001": {
 						CveID: "CVE-2017-0001",
@@ -246,7 +247,10 @@ func TestFilterIgnoreCveIDs(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		actual := tt.in.rs.FilterIgnoreCves(tt.in.cves)
+		config.Conf.Servers = map[string]config.ServerInfo{
+			"name": {IgnoreCves: tt.in.cves},
+		}
+		actual := tt.in.rs.FilterIgnoreCves()
 		for k := range tt.out.ScannedCves {
 			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
 				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
@@ -254,6 +258,83 @@ func TestFilterIgnoreCveIDs(t *testing.T) {
 				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
 			}
 		}
+		for k := range actual.ScannedCves {
+			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
+				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
+				a := pp.Sprintf("%v", actual.ScannedCves[k])
+				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
+			}
+		}
+	}
+}
+
+func TestFilterIgnoreCveIDsContainer(t *testing.T) {
+	type in struct {
+		cves []string
+		rs   ScanResult
+	}
+	var tests = []struct {
+		in  in
+		out ScanResult
+	}{
+		{
+			in: in{
+				cves: []string{"CVE-2017-0002"},
+				rs: ScanResult{
+					ServerName: "name",
+					Container:  Container{Name: "dockerA"},
+					ScannedCves: VulnInfos{
+						"CVE-2017-0001": {
+							CveID: "CVE-2017-0001",
+						},
+						"CVE-2017-0002": {
+							CveID: "CVE-2017-0002",
+						},
+						"CVE-2017-0003": {
+							CveID: "CVE-2017-0003",
+						},
+					},
+				},
+			},
+			out: ScanResult{
+				ServerName: "name",
+				Container:  Container{Name: "dockerA"},
+				ScannedCves: VulnInfos{
+					"CVE-2017-0001": {
+						CveID: "CVE-2017-0001",
+					},
+					"CVE-2017-0003": {
+						CveID: "CVE-2017-0003",
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		config.Conf.Servers = map[string]config.ServerInfo{
+			"name": {
+				Containers: map[string]config.ContainerSetting{
+					"dockerA": {
+						IgnoreCves: tt.in.cves,
+					},
+				},
+			},
+		}
+		actual := tt.in.rs.FilterIgnoreCves()
+		for k := range tt.out.ScannedCves {
+			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
+				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
+				a := pp.Sprintf("%v", actual.ScannedCves[k])
+				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
+			}
+		}
+		for k := range actual.ScannedCves {
+			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
+				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
+				a := pp.Sprintf("%v", actual.ScannedCves[k])
+				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
+			}
+		}
 	}
 }
 
@@ -336,3 +417,322 @@ func TestFilterUnfixed(t *testing.T) {
 		}
 	}
 }
+
+func TestFilterIgnorePkgs(t *testing.T) {
+	type in struct {
+		ignorePkgsRegexp []string
+		rs               ScanResult
+	}
+	var tests = []struct {
+		in  in
+		out ScanResult
+	}{
+		{
+			in: in{
+				ignorePkgsRegexp: []string{"^kernel"},
+				rs: ScanResult{
+					ServerName: "name",
+					ScannedCves: VulnInfos{
+						"CVE-2017-0001": {
+							CveID: "CVE-2017-0001",
+							AffectedPackages: PackageStatuses{
+								{Name: "kernel"},
+							},
+						},
+						"CVE-2017-0002": {
+							CveID: "CVE-2017-0002",
+						},
+					},
+				},
+			},
+			out: ScanResult{
+				ServerName: "name",
+				ScannedCves: VulnInfos{
+					"CVE-2017-0002": {
+						CveID: "CVE-2017-0002",
+					},
+				},
+			},
+		},
+		{
+			in: in{
+				ignorePkgsRegexp: []string{"^kernel"},
+				rs: ScanResult{
+					ServerName: "name",
+					ScannedCves: VulnInfos{
+						"CVE-2017-0001": {
+							CveID: "CVE-2017-0001",
+							AffectedPackages: PackageStatuses{
+								{Name: "kernel"},
+								{Name: "vim"},
+							},
+						},
+					},
+				},
+			},
+			out: ScanResult{
+				ServerName: "name",
+				ScannedCves: VulnInfos{
+					"CVE-2017-0001": {
+						CveID: "CVE-2017-0001",
+						AffectedPackages: PackageStatuses{
+							{Name: "kernel"},
+							{Name: "vim"},
+						},
+					},
+				},
+			},
+		},
+		{
+			in: in{
+				ignorePkgsRegexp: []string{"^kernel", "^vim", "^bind"},
+				rs: ScanResult{
+					ServerName: "name",
+					ScannedCves: VulnInfos{
+						"CVE-2017-0001": {
+							CveID: "CVE-2017-0001",
+							AffectedPackages: PackageStatuses{
+								{Name: "kernel"},
+								{Name: "vim"},
+							},
+						},
+					},
+				},
+			},
+			out: ScanResult{
+				ServerName:  "name",
+				ScannedCves: VulnInfos{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		config.Conf.Servers = map[string]config.ServerInfo{
+			"name": {IgnorePkgsRegexp: tt.in.ignorePkgsRegexp},
+		}
+		actual := tt.in.rs.FilterIgnorePkgs()
+		for k := range tt.out.ScannedCves {
+			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
+				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
+				a := pp.Sprintf("%v", actual.ScannedCves[k])
+				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
+			}
+		}
+		for k := range actual.ScannedCves {
+			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
+				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
+				a := pp.Sprintf("%v", actual.ScannedCves[k])
+				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
+			}
+		}
+	}
+}
+
+func TestFilterIgnorePkgsContainer(t *testing.T) {
+	type in struct {
+		ignorePkgsRegexp []string
+		rs               ScanResult
+	}
+	var tests = []struct {
+		in  in
+		out ScanResult
+	}{
+		{
+			in: in{
+				ignorePkgsRegexp: []string{"^kernel"},
+				rs: ScanResult{
+					ServerName: "name",
+					Container:  Container{Name: "dockerA"},
+					ScannedCves: VulnInfos{
+						"CVE-2017-0001": {
+							CveID: "CVE-2017-0001",
+							AffectedPackages: PackageStatuses{
+								{Name: "kernel"},
+							},
+						},
+						"CVE-2017-0002": {
+							CveID: "CVE-2017-0002",
+						},
+					},
+				},
+			},
+			out: ScanResult{
+				ServerName: "name",
+				Container:  Container{Name: "dockerA"},
+				ScannedCves: VulnInfos{
+					"CVE-2017-0002": {
+						CveID: "CVE-2017-0002",
+					},
+				},
+			},
+		},
+		{
+			in: in{
+				ignorePkgsRegexp: []string{"^kernel"},
+				rs: ScanResult{
+					ServerName: "name",
+					Container:  Container{Name: "dockerA"},
+					ScannedCves: VulnInfos{
+						"CVE-2017-0001": {
+							CveID: "CVE-2017-0001",
+							AffectedPackages: PackageStatuses{
+								{Name: "kernel"},
+								{Name: "vim"},
+							},
+						},
+					},
+				},
+			},
+			out: ScanResult{
+				ServerName: "name",
+				Container:  Container{Name: "dockerA"},
+				ScannedCves: VulnInfos{
+					"CVE-2017-0001": {
+						CveID: "CVE-2017-0001",
+						AffectedPackages: PackageStatuses{
+							{Name: "kernel"},
+							{Name: "vim"},
+						},
+					},
+				},
+			},
+		},
+		{
+			in: in{
+				ignorePkgsRegexp: []string{"^kernel", "^vim", "^bind"},
+				rs: ScanResult{
+					ServerName: "name",
+					Container:  Container{Name: "dockerA"},
+					ScannedCves: VulnInfos{
+						"CVE-2017-0001": {
+							CveID: "CVE-2017-0001",
+							AffectedPackages: PackageStatuses{
+								{Name: "kernel"},
+								{Name: "vim"},
+							},
+						},
+					},
+				},
+			},
+			out: ScanResult{
+				ServerName:  "name",
+				Container:   Container{Name: "dockerA"},
+				ScannedCves: VulnInfos{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		config.Conf.Servers = map[string]config.ServerInfo{
+			"name": {
+				Containers: map[string]config.ContainerSetting{
+					"dockerA": {
+						IgnorePkgsRegexp: tt.in.ignorePkgsRegexp,
+					},
+				},
+			},
+		}
+		actual := tt.in.rs.FilterIgnorePkgs()
+		for k := range tt.out.ScannedCves {
+			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
+				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
+				a := pp.Sprintf("%v", actual.ScannedCves[k])
+				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
+			}
+		}
+		for k := range actual.ScannedCves {
+			if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) {
+				o := pp.Sprintf("%v", tt.out.ScannedCves[k])
+				a := pp.Sprintf("%v", actual.ScannedCves[k])
+				t.Errorf("[%s] expected: %v\n  actual: %v\n", k, o, a)
+			}
+		}
+	}
+}
+
+func TestIsDisplayUpdatableNum(t *testing.T) {
+	var tests = []struct {
+		mode     []byte
+		family   string
+		expected bool
+	}{
+		{
+			mode:     []byte{config.Offline},
+			expected: false,
+		},
+		{
+			mode:     []byte{config.FastRoot},
+			expected: true,
+		},
+		{
+			mode:     []byte{config.Deep},
+			expected: true,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.RedHat,
+			expected: false,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.Oracle,
+			expected: false,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.Debian,
+			expected: false,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.Ubuntu,
+			expected: false,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.Raspbian,
+			expected: false,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.CentOS,
+			expected: true,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.Amazon,
+			expected: true,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.FreeBSD,
+			expected: true,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.OpenSUSE,
+			expected: true,
+		},
+		{
+			mode:     []byte{config.Fast},
+			family:   config.Alpine,
+			expected: true,
+		},
+	}
+
+	for i, tt := range tests {
+		mode := config.ScanMode{}
+		for _, m := range tt.mode {
+			mode.Set(m)
+		}
+		config.Conf.Servers = map[string]config.ServerInfo{
+			"name": {Mode: mode},
+		}
+		r := ScanResult{
+			ServerName: "name",
+			Family:     tt.family,
+		}
+		act := r.isDisplayUpdatableNum()
+		if tt.expected != act {
+			t.Errorf("[%d] expected %#v, actual %#v", i, tt.expected, act)
+		}
+	}
+}

models/utils.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,17 +18,22 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package models
 
 import (
-	"fmt"
 	"strings"
 
 	cvedict "github.com/kotakanbe/go-cve-dictionary/models"
 )
 
-// ConvertNvdToModel convert NVD to CveContent
-func ConvertNvdToModel(cveID string, nvd cvedict.Nvd) *CveContent {
+// ConvertNvdXMLToModel convert NVD to CveContent
+func ConvertNvdXMLToModel(cveID string, nvd *cvedict.NvdXML) *CveContent {
+	if nvd == nil {
+		return nil
+	}
 	var cpes []Cpe
 	for _, c := range nvd.Cpes {
-		cpes = append(cpes, Cpe{CpeName: c.CpeName})
+		cpes = append(cpes, Cpe{
+			FormattedString: c.FormattedString,
+			URI:             c.URI,
+		})
 	}
 
 	var refs []Reference
@@ -39,42 +44,21 @@ func ConvertNvdToModel(cveID string, nvd cvedict.Nvd) *CveContent {
 		})
 	}
 
-	validVec := true
-	for _, v := range []string{
-		nvd.AccessVector,
-		nvd.AccessComplexity,
-		nvd.Authentication,
-		nvd.ConfidentialityImpact,
-		nvd.IntegrityImpact,
-		nvd.AvailabilityImpact,
-	} {
-		if len(v) == 0 {
-			validVec = false
-		}
+	cweIDs := []string{}
+	for _, cid := range nvd.Cwes {
+		cweIDs = append(cweIDs, cid.CweID)
 	}
 
-	vector := ""
-	if validVec {
-		vector = fmt.Sprintf("AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s",
-			string(nvd.AccessVector[0]),
-			string(nvd.AccessComplexity[0]),
-			string(nvd.Authentication[0]),
-			string(nvd.ConfidentialityImpact[0]),
-			string(nvd.IntegrityImpact[0]),
-			string(nvd.AvailabilityImpact[0]))
-	}
-
-	//TODO CVSSv3
 	return &CveContent{
-		Type:         NVD,
-		CveID:        cveID,
-		Summary:      nvd.Summary,
-		Cvss2Score:   nvd.Score,
-		Cvss2Vector:  vector,
-		Severity:     "", // severity is not contained in NVD
-		SourceLink:   "https://nvd.nist.gov/vuln/detail/" + cveID,
-		Cpes:         cpes,
-		CweID:        nvd.CweID,
+		Type:          Nvd,
+		CveID:         cveID,
+		Summary:       nvd.Summary,
+		Cvss2Score:    nvd.Cvss2.BaseScore,
+		Cvss2Vector:   nvd.Cvss2.VectorString,
+		Cvss2Severity: nvd.Cvss2.Severity,
+		SourceLink:    "https://nvd.nist.gov/vuln/detail/" + cveID,
+		// Cpes:          cpes,
+		CweIDs:       cweIDs,
 		References:   refs,
 		Published:    nvd.PublishedDate,
 		LastModified: nvd.LastModifiedDate,
@@ -82,10 +66,16 @@ func ConvertNvdToModel(cveID string, nvd cvedict.Nvd) *CveContent {
 }
 
 // ConvertJvnToModel convert JVN to CveContent
-func ConvertJvnToModel(cveID string, jvn cvedict.Jvn) *CveContent {
+func ConvertJvnToModel(cveID string, jvn *cvedict.Jvn) *CveContent {
+	if jvn == nil {
+		return nil
+	}
 	var cpes []Cpe
 	for _, c := range jvn.Cpes {
-		cpes = append(cpes, Cpe{CpeName: c.CpeName})
+		cpes = append(cpes, Cpe{
+			FormattedString: c.FormattedString,
+			URI:             c.URI,
+		})
 	}
 
 	refs := []Reference{}
@@ -96,19 +86,71 @@ func ConvertJvnToModel(cveID string, jvn cvedict.Jvn) *CveContent {
 		})
 	}
 
-	vector := strings.TrimSuffix(strings.TrimPrefix(jvn.Vector, "("), ")")
 	return &CveContent{
-		Type:         JVN,
-		CveID:        cveID,
-		Title:        jvn.Title,
-		Summary:      jvn.Summary,
-		Severity:     jvn.Severity,
-		Cvss2Score:   jvn.Score,
-		Cvss2Vector:  vector,
-		SourceLink:   jvn.JvnLink,
-		Cpes:         cpes,
+		Type:          Jvn,
+		CveID:         cveID,
+		Title:         jvn.Title,
+		Summary:       jvn.Summary,
+		Cvss2Score:    jvn.Cvss2.BaseScore,
+		Cvss2Vector:   jvn.Cvss2.VectorString,
+		Cvss2Severity: jvn.Cvss2.Severity,
+		Cvss3Score:    jvn.Cvss3.BaseScore,
+		Cvss3Vector:   jvn.Cvss3.VectorString,
+		Cvss3Severity: jvn.Cvss3.BaseSeverity,
+		SourceLink:    jvn.JvnLink,
+		// Cpes:          cpes,
 		References:   refs,
 		Published:    jvn.PublishedDate,
 		LastModified: jvn.LastModifiedDate,
 	}
 }
+
+// ConvertNvdJSONToModel convert NVD to CveContent
+func ConvertNvdJSONToModel(cveID string, nvd *cvedict.NvdJSON) *CveContent {
+	if nvd == nil {
+		return nil
+	}
+	var cpes []Cpe
+	for _, c := range nvd.Cpes {
+		cpes = append(cpes, Cpe{
+			FormattedString: c.FormattedString,
+			URI:             c.URI,
+		})
+	}
+
+	var refs []Reference
+	for _, r := range nvd.References {
+		refs = append(refs, Reference{
+			Link:   r.Link,
+			Source: r.Source,
+		})
+	}
+
+	cweIDs := []string{}
+	for _, cid := range nvd.Cwes {
+		cweIDs = append(cweIDs, cid.CweID)
+	}
+
+	desc := []string{}
+	for _, d := range nvd.Descriptions {
+		desc = append(desc, d.Value)
+	}
+
+	return &CveContent{
+		Type:          Nvd,
+		CveID:         cveID,
+		Summary:       strings.Join(desc, "\n"),
+		Cvss2Score:    nvd.Cvss2.BaseScore,
+		Cvss2Vector:   nvd.Cvss2.VectorString,
+		Cvss2Severity: nvd.Cvss2.Severity,
+		Cvss3Score:    nvd.Cvss3.BaseScore,
+		Cvss3Vector:   nvd.Cvss3.VectorString,
+		Cvss3Severity: nvd.Cvss3.BaseSeverity,
+		SourceLink:    "https://nvd.nist.gov/vuln/detail/" + cveID,
+		// Cpes:          cpes,
+		CweIDs:       cweIDs,
+		References:   refs,
+		Published:    nvd.PublishedDate,
+		LastModified: nvd.LastModifiedDate,
+	}
+}

models/vulninfos.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/future-architect/vuls/config"
+	exploitmodels "github.com/mozqnet/go-exploitdb/models"
 )
 
 // VulnInfos has a map of VulnInfo
@@ -104,43 +105,86 @@ func (v VulnInfos) FormatCveSummary() string {
 		m["High"], m["Medium"], m["Low"], m["Unknown"])
 }
 
+// FormatFixedStatus summarize the number of cves are fixed.
+func (v VulnInfos) FormatFixedStatus(packs Packages) string {
+	total, fixed := 0, 0
+	for _, vInfo := range v {
+		if len(vInfo.CpeURIs) != 0 {
+			continue
+		}
+		total++
+		if vInfo.PatchStatus(packs) == "Fixed" {
+			fixed++
+		}
+	}
+	return fmt.Sprintf("%d/%d Fixed", fixed, total)
+}
+
 // PackageStatuses is a list of PackageStatus
 type PackageStatuses []PackageStatus
 
+// FormatTuiSummary format packname to show TUI summary
+func (ps PackageStatuses) FormatTuiSummary() string {
+	names := []string{}
+	for _, p := range ps {
+		names = append(names, p.Name)
+	}
+	return strings.Join(names, ", ")
+}
+
+// Store insert given pkg if missing, update pkg if exists
+func (ps PackageStatuses) Store(pkg PackageStatus) PackageStatuses {
+	for i, p := range ps {
+		if p.Name == pkg.Name {
+			ps[i] = pkg
+			return ps
+		}
+	}
+	ps = append(ps, pkg)
+	return ps
+}
+
 // Sort by Name
-func (p PackageStatuses) Sort() {
-	sort.Slice(p, func(i, j int) bool {
-		return p[i].Name < p[j].Name
+func (ps PackageStatuses) Sort() {
+	sort.Slice(ps, func(i, j int) bool {
+		return ps[i].Name < ps[j].Name
 	})
 	return
 }
 
 // PackageStatus has name and other status abount the package
 type PackageStatus struct {
-	Name        string
-	NotFixedYet bool
+	Name        string `json:"name"`
+	NotFixedYet bool   `json:"notFixedYet"`
+	FixState    string `json:"fixState"`
 }
 
 // VulnInfo has a vulnerability information and unsecure packages
 type VulnInfo struct {
-	CveID            string
-	Confidence       Confidence
-	AffectedPackages PackageStatuses
-	DistroAdvisories []DistroAdvisory // for Aamazon, RHEL, FreeBSD
-	CpeNames         []string
-	CveContents      CveContents
+	CveID            string           `json:"cveID"`
+	Confidences      Confidences      `json:"confidences"`
+	AffectedPackages PackageStatuses  `json:"affectedPackages"`
+	DistroAdvisories []DistroAdvisory `json:"distroAdvisories,omitempty"` // for Aamazon, RHEL, FreeBSD
+	CpeURIs          []string         `json:"cpeURIs,omitempty"`          // CpeURIs related to this CVE defined in config.toml
+	CveContents      CveContents      `json:"cveContents"`
+	Exploits         []Exploit        `json:"exploits"`
 }
 
 // Titles returns tilte (TUI)
 func (v VulnInfo) Titles(lang, myFamily string) (values []CveContentStr) {
 	if lang == "ja" {
-		if cont, found := v.CveContents[JVN]; found && 0 < len(cont.Title) {
-			values = append(values, CveContentStr{JVN, cont.Title})
+		if cont, found := v.CveContents[Jvn]; found && 0 < len(cont.Title) {
+			values = append(values, CveContentStr{Jvn, cont.Title})
 		}
 	}
 
-	order := CveContentTypes{NVD, NewCveContentType(myFamily)}
-	order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...)
+	// RedHat API has one line title.
+	if cont, found := v.CveContents[RedHatAPI]; found && 0 < len(cont.Title) {
+		values = append(values, CveContentStr{RedHatAPI, cont.Title})
+	}
+
+	order := CveContentTypes{Nvd, NvdXML, NewCveContentType(myFamily)}
+	order = append(order, AllCveContetTypes.Except(append(order, Jvn)...)...)
 	for _, ctype := range order {
 		// Only JVN has meaningful title. so return first 100 char of summary
 		if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Summary) {
@@ -171,16 +215,16 @@ func (v VulnInfo) Titles(lang, myFamily string) (values []CveContentStr) {
 // Summaries returns summaries
 func (v VulnInfo) Summaries(lang, myFamily string) (values []CveContentStr) {
 	if lang == "ja" {
-		if cont, found := v.CveContents[JVN]; found && 0 < len(cont.Summary) {
+		if cont, found := v.CveContents[Jvn]; found && 0 < len(cont.Summary) {
 			summary := cont.Title
 			summary += "\n" + strings.Replace(
 				strings.Replace(cont.Summary, "\n", " ", -1), "\r", " ", -1)
-			values = append(values, CveContentStr{JVN, summary})
+			values = append(values, CveContentStr{Jvn, summary})
 		}
 	}
 
-	order := CveContentTypes{NVD, NewCveContentType(myFamily)}
-	order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...)
+	order := CveContentTypes{Nvd, NvdXML, NewCveContentType(myFamily)}
+	order = append(order, AllCveContetTypes.Except(append(order, Jvn)...)...)
 	for _, ctype := range order {
 		if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Summary) {
 			summary := strings.Replace(cont.Summary, "\n", " ", -1)
@@ -208,32 +252,75 @@ func (v VulnInfo) Summaries(lang, myFamily string) (values []CveContentStr) {
 	return
 }
 
-// Cvss2Scores returns CVSS V2 Scores
-func (v VulnInfo) Cvss2Scores() (values []CveContentCvss) {
-	order := []CveContentType{NVD, RedHat, JVN}
+// Mitigations returns mitigations
+func (v VulnInfo) Mitigations(myFamily string) (values []CveContentStr) {
+	order := CveContentTypes{RedHatAPI}
 	for _, ctype := range order {
-		if cont, found := v.CveContents[ctype]; found && 0 < cont.Cvss2Score {
-			// https://nvd.nist.gov/vuln-metrics/cvss
-			sev := cont.Severity
-			if ctype == NVD {
-				sev = cvss2ScoreToSeverity(cont.Cvss2Score)
+		if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Mitigation) {
+			values = append(values, CveContentStr{
+				Type:  ctype,
+				Value: cont.Mitigation,
+			})
+		}
+	}
+
+	if len(values) == 0 {
+		return []CveContentStr{{
+			Type:  Unknown,
+			Value: "-",
+		}}
+	}
+	return
+}
+
+// Cvss2Scores returns CVSS V2 Scores
+func (v VulnInfo) Cvss2Scores(myFamily string) (values []CveContentCvss) {
+	order := []CveContentType{Nvd, NvdXML, RedHat, Jvn}
+	if myFamily != config.RedHat && myFamily != config.CentOS {
+		order = append(order, NewCveContentType(myFamily))
+	}
+	for _, ctype := range order {
+		if cont, found := v.CveContents[ctype]; found {
+			if cont.Cvss2Score == 0 && cont.Cvss2Severity == "" {
+				continue
 			}
+			// https://nvd.nist.gov/vuln-metrics/cvss
 			values = append(values, CveContentCvss{
 				Type: ctype,
 				Value: Cvss{
 					Type:     CVSS2,
 					Score:    cont.Cvss2Score,
 					Vector:   cont.Cvss2Vector,
-					Severity: strings.ToUpper(sev),
+					Severity: strings.ToUpper(cont.Cvss2Severity),
 				},
 			})
 		}
 	}
 
+	for _, v := range values {
+		if v.Type == RedHat {
+			return
+		}
+	}
+	// Set the CVSS v2 score of vuln that exists only in gost.
+	// Unfixed vulnerabilities detected by gost are not in OVAL, because
+	// OVAL data has only vulnerabilities for already fixed.
+	if cont, found := v.CveContents[RedHatAPI]; found {
+		values = append(values, CveContentCvss{
+			Type: RedHatAPI,
+			Value: Cvss{
+				Type:     CVSS2,
+				Score:    cont.Cvss2Score,
+				Vector:   cont.Cvss2Vector,
+				Severity: strings.ToUpper(cont.Cvss2Severity),
+			},
+		})
+	}
+
 	for _, adv := range v.DistroAdvisories {
 		if adv.Severity != "" {
 			values = append(values, CveContentCvss{
-				Type: "Vendor",
+				Type: "Advisory",
 				Value: Cvss{
 					Type:                 CVSS2,
 					Score:                severityToV2ScoreRoughly(adv.Severity),
@@ -245,35 +332,75 @@ func (v VulnInfo) Cvss2Scores() (values []CveContentCvss) {
 		}
 	}
 
+	// An OVAL entry in Ubuntu and Debian has only severity (CVSS score isn't included).
+	// Show severity and dummy score calculated roughly.
+	order = append(order, AllCveContetTypes.Except(order...)...)
+	for _, ctype := range order {
+		if cont, found := v.CveContents[ctype]; found &&
+			cont.Cvss2Score == 0 &&
+			cont.Cvss3Score == 0 &&
+			cont.Cvss2Severity != "" {
+
+			values = append(values, CveContentCvss{
+				Type: cont.Type,
+				Value: Cvss{
+					Type:                 CVSS2,
+					Score:                severityToV2ScoreRoughly(cont.Cvss2Severity),
+					CalculatedBySeverity: true,
+					Vector:               "-",
+					Severity:             strings.ToUpper(cont.Cvss2Severity),
+				},
+			})
+		}
+	}
+
 	return
 }
 
 // Cvss3Scores returns CVSS V3 Score
 func (v VulnInfo) Cvss3Scores() (values []CveContentCvss) {
-	// TODO implement NVD
-	order := []CveContentType{RedHat}
+	order := []CveContentType{Nvd, RedHat, Jvn}
 	for _, ctype := range order {
-		if cont, found := v.CveContents[ctype]; found && 0 < cont.Cvss3Score {
+		if cont, found := v.CveContents[ctype]; found {
 			// https://nvd.nist.gov/vuln-metrics/cvss
-			sev := cont.Severity
 			values = append(values, CveContentCvss{
 				Type: ctype,
 				Value: Cvss{
 					Type:     CVSS3,
 					Score:    cont.Cvss3Score,
 					Vector:   cont.Cvss3Vector,
-					Severity: strings.ToUpper(sev),
+					Severity: strings.ToUpper(cont.Cvss3Severity),
 				},
 			})
 		}
 	}
+
+	for _, v := range values {
+		if v.Type == RedHat {
+			return
+		}
+	}
+
+	// Set the CVSS v3 score of vuln that exists only in gost.
+	// Unfixed vulnerabilities detected by gost are not in OVAL, because
+	// OVAL data has only vulnerabilities for already fixed.
+	if cont, found := v.CveContents[RedHatAPI]; found {
+		values = append(values, CveContentCvss{
+			Type: RedHatAPI,
+			Value: Cvss{
+				Type:     CVSS3,
+				Score:    cont.Cvss3Score,
+				Vector:   cont.Cvss3Vector,
+				Severity: strings.ToUpper(cont.Cvss3Severity),
+			},
+		})
+	}
 	return
 }
 
 // MaxCvss3Score returns Max CVSS V3 Score
 func (v VulnInfo) MaxCvss3Score() CveContentCvss {
-	// TODO implement NVD
-	order := []CveContentType{RedHat}
+	order := []CveContentType{Nvd, RedHat, RedHatAPI, Jvn}
 	max := 0.0
 	value := CveContentCvss{
 		Type:  Unknown,
@@ -282,14 +409,13 @@ func (v VulnInfo) MaxCvss3Score() CveContentCvss {
 	for _, ctype := range order {
 		if cont, found := v.CveContents[ctype]; found && max < cont.Cvss3Score {
 			// https://nvd.nist.gov/vuln-metrics/cvss
-			sev := cont.Severity
 			value = CveContentCvss{
 				Type: ctype,
 				Value: Cvss{
 					Type:     CVSS3,
 					Score:    cont.Cvss3Score,
 					Vector:   cont.Cvss3Vector,
-					Severity: sev,
+					Severity: strings.ToUpper(cont.Cvss3Severity),
 				},
 			}
 			max = cont.Cvss3Score
@@ -316,7 +442,7 @@ func (v VulnInfo) MaxCvssScore() CveContentCvss {
 
 // MaxCvss2Score returns Max CVSS V2 Score
 func (v VulnInfo) MaxCvss2Score() CveContentCvss {
-	order := []CveContentType{NVD, RedHat, JVN}
+	order := []CveContentType{Nvd, NvdXML, RedHat, RedHatAPI, Jvn}
 	max := 0.0
 	value := CveContentCvss{
 		Type:  Unknown,
@@ -325,17 +451,13 @@ func (v VulnInfo) MaxCvss2Score() CveContentCvss {
 	for _, ctype := range order {
 		if cont, found := v.CveContents[ctype]; found && max < cont.Cvss2Score {
 			// https://nvd.nist.gov/vuln-metrics/cvss
-			sev := cont.Severity
-			if ctype == NVD {
-				sev = cvss2ScoreToSeverity(cont.Cvss2Score)
-			}
 			value = CveContentCvss{
 				Type: ctype,
 				Value: Cvss{
 					Type:     CVSS2,
 					Score:    cont.Cvss2Score,
 					Vector:   cont.Cvss2Vector,
-					Severity: sev,
+					Severity: strings.ToUpper(cont.Cvss2Severity),
 				},
 			}
 			max = cont.Cvss2Score
@@ -350,8 +472,8 @@ func (v VulnInfo) MaxCvss2Score() CveContentCvss {
 	// Only Ubuntu, RedHat and Oracle have severity data in OVAL.
 	order = []CveContentType{Ubuntu, RedHat, Oracle}
 	for _, ctype := range order {
-		if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Severity) {
-			score := severityToV2ScoreRoughly(cont.Severity)
+		if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Cvss2Severity) {
+			score := severityToV2ScoreRoughly(cont.Cvss2Severity)
 			if max < score {
 				value = CveContentCvss{
 					Type: ctype,
@@ -360,7 +482,7 @@ func (v VulnInfo) MaxCvss2Score() CveContentCvss {
 						Score:                score,
 						CalculatedBySeverity: true,
 						Vector:               cont.Cvss2Vector,
-						Severity:             cont.Severity,
+						Severity:             strings.ToUpper(cont.Cvss2Severity),
 					},
 				}
 			}
@@ -389,10 +511,55 @@ func (v VulnInfo) MaxCvss2Score() CveContentCvss {
 	return value
 }
 
-// CveContentCvss has CveContentType and Cvss2
+// AttackVector returns attack vector string
+func (v VulnInfo) AttackVector() string {
+	for _, cnt := range v.CveContents {
+		if strings.HasPrefix(cnt.Cvss2Vector, "AV:N") ||
+			strings.HasPrefix(cnt.Cvss3Vector, "CVSS:3.0/AV:N") {
+			return "Network"
+		} else if strings.HasPrefix(cnt.Cvss2Vector, "AV:A") ||
+			strings.HasPrefix(cnt.Cvss3Vector, "CVSS:3.0/AV:A") {
+			return "Adjacent"
+		} else if strings.HasPrefix(cnt.Cvss2Vector, "AV:L") ||
+			strings.HasPrefix(cnt.Cvss3Vector, "CVSS:3.0/AV:L") {
+			return "Local"
+		} else if strings.HasPrefix(cnt.Cvss3Vector, "CVSS:3.0/AV:P") {
+			return "Physical"
+		}
+	}
+	if cont, found := v.CveContents[DebianSecurityTracker]; found {
+		if attackRange, found := cont.Optional["attack range"]; found {
+			return attackRange
+		}
+	}
+	return ""
+}
+
+// PatchStatus returns attack vector string
+func (v VulnInfo) PatchStatus(packs Packages) string {
+	// Vuls don't know patch status of the CPE
+	if len(v.CpeURIs) != 0 {
+		return ""
+	}
+	for _, p := range v.AffectedPackages {
+		if p.NotFixedYet {
+			return "Unfixed"
+		}
+
+		// fast, offline mode doesn't have new version
+		if pack, ok := packs[p.Name]; ok {
+			if pack.NewVersion == "" {
+				return "Unknown"
+			}
+		}
+	}
+	return "Fixed"
+}
+
+// CveContentCvss has CVSS information
 type CveContentCvss struct {
-	Type  CveContentType
-	Value Cvss
+	Type  CveContentType `json:"type"`
+	Value Cvss           `json:"value"`
 }
 
 // CvssType Represent the type of CVSS
@@ -408,20 +575,23 @@ const (
 
 // Cvss has CVSS Score
 type Cvss struct {
-	Type                 CvssType
-	Score                float64
-	CalculatedBySeverity bool
-	Vector               string
-	Severity             string
+	Type                 CvssType `json:"type"`
+	Score                float64  `json:"score"`
+	CalculatedBySeverity bool     `json:"calculatedBySeverity"`
+	Vector               string   `json:"vector"`
+	Severity             string   `json:"severity"`
 }
 
 // Format CVSS Score and Vector
 func (c Cvss) Format() string {
+	if c.Score == 0 || c.Vector == "" {
+		return c.Severity
+	}
 	switch c.Type {
 	case CVSS2:
-		return fmt.Sprintf("%3.1f/%s", c.Score, c.Vector)
+		return fmt.Sprintf("%3.1f/%s %s", c.Score, c.Vector, c.Severity)
 	case CVSS3:
-		return fmt.Sprintf("%3.1f/CVSS:3.0/%s", c.Score, c.Vector)
+		return fmt.Sprintf("%3.1f/%s %s", c.Score, c.Vector, c.Severity)
 	}
 	return ""
 }
@@ -461,49 +631,13 @@ func severityToV2ScoreRoughly(severity string) float64 {
 	return 0
 }
 
-// CveContentCvss3 has CveContentType and Cvss3
-//  type CveContentCvss3 struct {
-//      Type  CveContentType
-//      Value Cvss3
-//  }
-
-// Cvss3 has CVSS v3 Score, Vector and  Severity
-//  type Cvss3 struct {
-//      Score    float64
-//      Vector   string
-//      Severity string
-//  }
-
-// Format CVSS Score and Vector
-//  func (c Cvss3) Format() string {
-//      return fmt.Sprintf("%3.1f/CVSS:3.0/%s", c.Score, c.Vector)
-//  }
-
-//  func cvss3ScoreToSeverity(score float64) string {
-//      if 9.0 <= score {
-//          return "CRITICAL"
-//      } else if 7.0 <= score {
-//          return "HIGH"
-//      } else if 4.0 <= score {
-//          return "MEDIUM"
-//      }
-//      return "LOW"
-//  }
-
 // FormatMaxCvssScore returns Max CVSS Score
 func (v VulnInfo) FormatMaxCvssScore() string {
-	v2Max := v.MaxCvss2Score()
-	v3Max := v.MaxCvss3Score()
-	if v2Max.Value.Score <= v3Max.Value.Score {
-		return fmt.Sprintf("%3.1f %s (%s)",
-			v3Max.Value.Score,
-			strings.ToUpper(v3Max.Value.Severity),
-			v3Max.Type)
-	}
+	max := v.MaxCvssScore()
 	return fmt.Sprintf("%3.1f %s (%s)",
-		v2Max.Value.Score,
-		strings.ToUpper(v2Max.Value.Severity),
-		v2Max.Type)
+		max.Value.Score,
+		strings.ToUpper(max.Value.Severity),
+		max.Type)
 }
 
 // Cvss2CalcURL returns CVSS v2 caluclator's URL
@@ -558,37 +692,13 @@ func (v VulnInfo) VendorLinks(family string) map[string]string {
 	return links
 }
 
-// NilToEmpty set nil slice or map fields to empty to avoid null in JSON
-func (v *VulnInfo) NilToEmpty() *VulnInfo {
-	if v.CpeNames == nil {
-		v.CpeNames = []string{}
-	}
-	if v.DistroAdvisories == nil {
-		v.DistroAdvisories = []DistroAdvisory{}
-	}
-	if v.AffectedPackages == nil {
-		v.AffectedPackages = PackageStatuses{}
-	}
-	if v.CveContents == nil {
-		v.CveContents = NewCveContents()
-	}
-	for key := range v.CveContents {
-		if v.CveContents[key].Cpes == nil {
-			cont := v.CveContents[key]
-			cont.Cpes = []Cpe{}
-			v.CveContents[key] = cont
-		}
-	}
-	return v
-}
-
 // DistroAdvisory has Amazon Linux, RHEL, FreeBSD Security Advisory information.
 type DistroAdvisory struct {
-	AdvisoryID  string
-	Severity    string
-	Issued      time.Time
-	Updated     time.Time
-	Description string
+	AdvisoryID  string    `json:"advisoryID"`
+	Severity    string    `json:"severity"`
+	Issued      time.Time `json:"issued"`
+	Updated     time.Time `json:"updated"`
+	Description string    `json:"description"`
 }
 
 // Format the distro advisory information
@@ -605,11 +715,45 @@ func (p DistroAdvisory) Format() string {
 	return strings.Join(buf, "\n")
 }
 
+// Exploit :
+type Exploit struct {
+	ExploitType  exploitmodels.ExploitType `json:"exploitType"`
+	ID           string                    `json:"id"`
+	URL          string                    `json:"url"`
+	Description  string                    `json:"description"`
+	DocumentURL  *string                   `json:"documentURL,omitempty"`
+	PaperURL     *string                   `json:"paperURL,omitempty"`
+	ShellCodeURL *string                   `json:"shellCodeURL,omitempty"`
+	BinaryURL    *string                   `json:"binaryURL,omitempty"`
+}
+
+// Confidences is a list of Confidence
+type Confidences []Confidence
+
+// AppendIfMissing appends confidence to the list if missiong
+func (cs *Confidences) AppendIfMissing(confidence Confidence) {
+	for _, c := range *cs {
+		if c.DetectionMethod == confidence.DetectionMethod {
+			return
+		}
+	}
+	*cs = append(*cs, confidence)
+}
+
+// SortByConfident sorts Confidences
+func (cs Confidences) SortByConfident() Confidences {
+	sort.Slice(cs, func(i, j int) bool {
+		return cs[i].SortOrder < cs[j].SortOrder
+	})
+	return cs
+}
+
 // Confidence is a ranking how confident the CVE-ID was deteted correctly
 // Score: 0 - 100
 type Confidence struct {
-	Score           int
-	DetectionMethod DetectionMethod
+	Score           int             `json:"score"`
+	DetectionMethod DetectionMethod `json:"detectionMethod"`
+	SortOrder       int             `json:"-"`
 }
 
 func (c Confidence) String() string {
@@ -634,6 +778,12 @@ const (
 	// OvalMatchStr is a String representation of OvalMatch
 	OvalMatchStr = "OvalMatch"
 
+	// RedHatAPIStr is a String representation of RedHatAPIMatch
+	RedHatAPIStr = "RedHatAPIMatch"
+
+	// DebianSecurityTrackerMatchStr is a String representation of DebianSecurityTrackerMatch
+	DebianSecurityTrackerMatchStr = "DebianSecurityTrackerMatch"
+
 	// ChangelogExactMatchStr is a String representation of ChangelogExactMatch
 	ChangelogExactMatchStr = "ChangelogExactMatch"
 
@@ -649,20 +799,26 @@ const (
 
 var (
 	// CpeNameMatch is a ranking how confident the CVE-ID was deteted correctly
-	CpeNameMatch = Confidence{100, CpeNameMatchStr}
+	CpeNameMatch = Confidence{100, CpeNameMatchStr, 1}
 
 	// YumUpdateSecurityMatch is a ranking how confident the CVE-ID was deteted correctly
-	YumUpdateSecurityMatch = Confidence{100, YumUpdateSecurityMatchStr}
+	YumUpdateSecurityMatch = Confidence{100, YumUpdateSecurityMatchStr, 2}
 
 	// PkgAuditMatch is a ranking how confident the CVE-ID was deteted correctly
-	PkgAuditMatch = Confidence{100, PkgAuditMatchStr}
+	PkgAuditMatch = Confidence{100, PkgAuditMatchStr, 2}
 
 	// OvalMatch is a ranking how confident the CVE-ID was deteted correctly
-	OvalMatch = Confidence{100, OvalMatchStr}
+	OvalMatch = Confidence{100, OvalMatchStr, 0}
+
+	// RedHatAPIMatch ranking how confident the CVE-ID was deteted correctly
+	RedHatAPIMatch = Confidence{100, RedHatAPIStr, 0}
+
+	// DebianSecurityTrackerMatch ranking how confident the CVE-ID was deteted correctly
+	DebianSecurityTrackerMatch = Confidence{100, DebianSecurityTrackerMatchStr, 0}
 
 	// ChangelogExactMatch is a ranking how confident the CVE-ID was deteted correctly
-	ChangelogExactMatch = Confidence{95, ChangelogExactMatchStr}
+	ChangelogExactMatch = Confidence{95, ChangelogExactMatchStr, 3}
 
 	// ChangelogLenientMatch is a ranking how confident the CVE-ID was deteted correctly
-	ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr}
+	ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr, 4}
 )

models/vulninfos_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -36,16 +36,16 @@ func TestTitles(t *testing.T) {
 				lang: "ja",
 				cont: VulnInfo{
 					CveContents: CveContents{
-						JVN: {
-							Type:  JVN,
+						Jvn: {
+							Type:  Jvn,
 							Title: "Title1",
 						},
 						RedHat: {
 							Type:    RedHat,
 							Summary: "Summary RedHat",
 						},
-						NVD: {
-							Type:    NVD,
+						NvdXML: {
+							Type:    NvdXML,
 							Summary: "Summary NVD",
 							// Severity is NIOT included in NVD
 						},
@@ -54,11 +54,11 @@ func TestTitles(t *testing.T) {
 			},
 			out: []CveContentStr{
 				{
-					Type:  JVN,
+					Type:  Jvn,
 					Value: "Title1",
 				},
 				{
-					Type:  NVD,
+					Type:  NvdXML,
 					Value: "Summary NVD",
 				},
 				{
@@ -73,16 +73,16 @@ func TestTitles(t *testing.T) {
 				lang: "en",
 				cont: VulnInfo{
 					CveContents: CveContents{
-						JVN: {
-							Type:  JVN,
+						Jvn: {
+							Type:  Jvn,
 							Title: "Title1",
 						},
 						RedHat: {
 							Type:    RedHat,
 							Summary: "Summary RedHat",
 						},
-						NVD: {
-							Type:    NVD,
+						NvdXML: {
+							Type:    NvdXML,
 							Summary: "Summary NVD",
 							// Severity is NIOT included in NVD
 						},
@@ -91,7 +91,7 @@ func TestTitles(t *testing.T) {
 			},
 			out: []CveContentStr{
 				{
-					Type:  NVD,
+					Type:  NvdXML,
 					Value: "Summary NVD",
 				},
 				{
@@ -137,8 +137,8 @@ func TestSummaries(t *testing.T) {
 				lang: "ja",
 				cont: VulnInfo{
 					CveContents: CveContents{
-						JVN: {
-							Type:    JVN,
+						Jvn: {
+							Type:    Jvn,
 							Title:   "Title JVN",
 							Summary: "Summary JVN",
 						},
@@ -146,8 +146,8 @@ func TestSummaries(t *testing.T) {
 							Type:    RedHat,
 							Summary: "Summary RedHat",
 						},
-						NVD: {
-							Type:    NVD,
+						NvdXML: {
+							Type:    NvdXML,
 							Summary: "Summary NVD",
 							// Severity is NIOT included in NVD
 						},
@@ -156,11 +156,11 @@ func TestSummaries(t *testing.T) {
 			},
 			out: []CveContentStr{
 				{
-					Type:  JVN,
+					Type:  Jvn,
 					Value: "Title JVN\nSummary JVN",
 				},
 				{
-					Type:  NVD,
+					Type:  NvdXML,
 					Value: "Summary NVD",
 				},
 				{
@@ -175,8 +175,8 @@ func TestSummaries(t *testing.T) {
 				lang: "en",
 				cont: VulnInfo{
 					CveContents: CveContents{
-						JVN: {
-							Type:    JVN,
+						Jvn: {
+							Type:    Jvn,
 							Title:   "Title JVN",
 							Summary: "Summary JVN",
 						},
@@ -184,8 +184,8 @@ func TestSummaries(t *testing.T) {
 							Type:    RedHat,
 							Summary: "Summary RedHat",
 						},
-						NVD: {
-							Type:    NVD,
+						NvdXML: {
+							Type:    NvdXML,
 							Summary: "Summary NVD",
 							// Severity is NIOT included in NVD
 						},
@@ -194,7 +194,7 @@ func TestSummaries(t *testing.T) {
 			},
 			out: []CveContentStr{
 				{
-					Type:  NVD,
+					Type:  NvdXML,
 					Value: "Summary NVD",
 				},
 				{
@@ -235,8 +235,8 @@ func TestCountGroupBySeverity(t *testing.T) {
 				"CVE-2017-0002": {
 					CveID: "CVE-2017-0002",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 6.0,
 						},
 						RedHat: {
@@ -248,8 +248,8 @@ func TestCountGroupBySeverity(t *testing.T) {
 				"CVE-2017-0003": {
 					CveID: "CVE-2017-0003",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 2.0,
 						},
 					},
@@ -257,8 +257,8 @@ func TestCountGroupBySeverity(t *testing.T) {
 				"CVE-2017-0004": {
 					CveID: "CVE-2017-0004",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 5.0,
 						},
 					},
@@ -296,8 +296,8 @@ func TestToSortedSlice(t *testing.T) {
 				"CVE-2017-0002": {
 					CveID: "CVE-2017-0002",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 6.0,
 						},
 						RedHat: {
@@ -309,8 +309,8 @@ func TestToSortedSlice(t *testing.T) {
 				"CVE-2017-0001": {
 					CveID: "CVE-2017-0001",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 7.0,
 						},
 						RedHat: {
@@ -324,8 +324,8 @@ func TestToSortedSlice(t *testing.T) {
 				{
 					CveID: "CVE-2017-0001",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 7.0,
 						},
 						RedHat: {
@@ -337,8 +337,8 @@ func TestToSortedSlice(t *testing.T) {
 				{
 					CveID: "CVE-2017-0002",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 6.0,
 						},
 						RedHat: {
@@ -355,8 +355,8 @@ func TestToSortedSlice(t *testing.T) {
 				"CVE-2017-0002": {
 					CveID: "CVE-2017-0002",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 6.0,
 						},
 						RedHat: {
@@ -388,8 +388,8 @@ func TestToSortedSlice(t *testing.T) {
 				{
 					CveID: "CVE-2017-0002",
 					CveContents: CveContents{
-						NVD: {
-							Type:       NVD,
+						NvdXML: {
+							Type:       NvdXML,
 							Cvss2Score: 6.0,
 						},
 						RedHat: {
@@ -407,8 +407,8 @@ func TestToSortedSlice(t *testing.T) {
 					CveID: "CVE-2017-0002",
 					CveContents: CveContents{
 						Ubuntu: {
-							Type:     Ubuntu,
-							Severity: "High",
+							Type:          Ubuntu,
+							Cvss2Severity: "High",
 						},
 					},
 				},
@@ -416,8 +416,8 @@ func TestToSortedSlice(t *testing.T) {
 					CveID: "CVE-2017-0001",
 					CveContents: CveContents{
 						Ubuntu: {
-							Type:     Ubuntu,
-							Severity: "Low",
+							Type:          Ubuntu,
+							Cvss2Severity: "Low",
 						},
 					},
 				},
@@ -427,8 +427,8 @@ func TestToSortedSlice(t *testing.T) {
 					CveID: "CVE-2017-0002",
 					CveContents: CveContents{
 						Ubuntu: {
-							Type:     Ubuntu,
-							Severity: "High",
+							Type:          Ubuntu,
+							Cvss2Severity: "High",
 						},
 					},
 				},
@@ -436,8 +436,8 @@ func TestToSortedSlice(t *testing.T) {
 					CveID: "CVE-2017-0001",
 					CveContents: CveContents{
 						Ubuntu: {
-							Type:     Ubuntu,
-							Severity: "Low",
+							Type:          Ubuntu,
+							Cvss2Severity: "Low",
 						},
 					},
 				},
@@ -460,29 +460,29 @@ func TestCvss2Scores(t *testing.T) {
 		{
 			in: VulnInfo{
 				CveContents: CveContents{
-					JVN: {
-						Type:        JVN,
-						Severity:    "HIGH",
-						Cvss2Score:  8.2,
-						Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P",
+					Jvn: {
+						Type:          Jvn,
+						Cvss2Severity: "HIGH",
+						Cvss2Score:    8.2,
+						Cvss2Vector:   "AV:N/AC:L/Au:N/C:N/I:N/A:P",
 					},
 					RedHat: {
-						Type:        RedHat,
-						Severity:    "HIGH",
-						Cvss2Score:  8.0,
-						Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P",
+						Type:          RedHat,
+						Cvss2Severity: "HIGH",
+						Cvss2Score:    8.0,
+						Cvss2Vector:   "AV:N/AC:L/Au:N/C:N/I:N/A:P",
 					},
-					NVD: {
-						Type:        NVD,
-						Cvss2Score:  8.1,
-						Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P",
-						// Severity is NIOT included in NVD
+					NvdXML: {
+						Type:          NvdXML,
+						Cvss2Score:    8.1,
+						Cvss2Vector:   "AV:N/AC:L/Au:N/C:N/I:N/A:P",
+						Cvss2Severity: "HIGH",
 					},
 				},
 			},
 			out: []CveContentCvss{
 				{
-					Type: NVD,
+					Type: NvdXML,
 					Value: Cvss{
 						Type:     CVSS2,
 						Score:    8.1,
@@ -500,7 +500,7 @@ func TestCvss2Scores(t *testing.T) {
 					},
 				},
 				{
-					Type: JVN,
+					Type: Jvn,
 					Value: Cvss{
 						Type:     CVSS2,
 						Score:    8.2,
@@ -517,9 +517,9 @@ func TestCvss2Scores(t *testing.T) {
 		},
 	}
 	for i, tt := range tests {
-		actual := tt.in.Cvss2Scores()
+		actual := tt.in.Cvss2Scores("redhat")
 		if !reflect.DeepEqual(tt.out, actual) {
-			t.Errorf("[%d] expected: %v\n  actual: %v\n", i, tt.out, actual)
+			t.Errorf("[%d]\nexpected: %v\n  actual: %v\n", i, tt.out, actual)
 		}
 	}
 }
@@ -532,20 +532,20 @@ func TestMaxCvss2Scores(t *testing.T) {
 		{
 			in: VulnInfo{
 				CveContents: CveContents{
-					JVN: {
-						Type:        JVN,
-						Severity:    "HIGH",
-						Cvss2Score:  8.2,
-						Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P",
+					Jvn: {
+						Type:          Jvn,
+						Cvss2Severity: "HIGH",
+						Cvss2Score:    8.2,
+						Cvss2Vector:   "AV:N/AC:L/Au:N/C:N/I:N/A:P",
 					},
 					RedHat: {
-						Type:        RedHat,
-						Severity:    "HIGH",
-						Cvss2Score:  8.0,
-						Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P",
+						Type:          RedHat,
+						Cvss2Severity: "HIGH",
+						Cvss2Score:    8.0,
+						Cvss2Vector:   "AV:N/AC:L/Au:N/C:N/I:N/A:P",
 					},
-					NVD: {
-						Type:        NVD,
+					NvdXML: {
+						Type:        NvdXML,
 						Cvss2Score:  8.1,
 						Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P",
 						// Severity is NIOT included in NVD
@@ -553,7 +553,7 @@ func TestMaxCvss2Scores(t *testing.T) {
 				},
 			},
 			out: CveContentCvss{
-				Type: JVN,
+				Type: Jvn,
 				Value: Cvss{
 					Type:     CVSS2,
 					Score:    8.2,
@@ -567,8 +567,8 @@ func TestMaxCvss2Scores(t *testing.T) {
 			in: VulnInfo{
 				CveContents: CveContents{
 					Ubuntu: {
-						Type:     Ubuntu,
-						Severity: "HIGH",
+						Type:          Ubuntu,
+						Cvss2Severity: "HIGH",
 					},
 				},
 			},
@@ -613,16 +613,16 @@ func TestCvss3Scores(t *testing.T) {
 			in: VulnInfo{
 				CveContents: CveContents{
 					RedHat: {
-						Type:        RedHat,
-						Severity:    "HIGH",
-						Cvss3Score:  8.0,
-						Cvss3Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
+						Type:          RedHat,
+						Cvss3Severity: "HIGH",
+						Cvss3Score:    8.0,
+						Cvss3Vector:   "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
 					},
-					NVD: {
-						Type:        NVD,
-						Cvss3Score:  8.1,
-						Cvss3Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
-						// Severity is NIOT included in NVD
+					NvdXML: {
+						Type:          NvdXML,
+						Cvss2Score:    8.1,
+						Cvss2Vector:   "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
+						Cvss2Severity: "HIGH",
 					},
 				},
 			},
@@ -661,10 +661,10 @@ func TestMaxCvss3Scores(t *testing.T) {
 			in: VulnInfo{
 				CveContents: CveContents{
 					RedHat: {
-						Type:        RedHat,
-						Severity:    "HIGH",
-						Cvss3Score:  8.0,
-						Cvss3Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
+						Type:          RedHat,
+						Cvss3Severity: "HIGH",
+						Cvss3Score:    8.0,
+						Cvss3Vector:   "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
 					},
 				},
 			},
@@ -708,8 +708,8 @@ func TestMaxCvssScores(t *testing.T) {
 		{
 			in: VulnInfo{
 				CveContents: CveContents{
-					NVD: {
-						Type:       NVD,
+					NvdXML: {
+						Type:       NvdXML,
 						Cvss3Score: 7.0,
 					},
 					RedHat: {
@@ -748,8 +748,8 @@ func TestMaxCvssScores(t *testing.T) {
 			in: VulnInfo{
 				CveContents: CveContents{
 					Ubuntu: {
-						Type:     Ubuntu,
-						Severity: "HIGH",
+						Type:          Ubuntu,
+						Cvss2Severity: "HIGH",
 					},
 				},
 			},
@@ -768,17 +768,18 @@ func TestMaxCvssScores(t *testing.T) {
 			in: VulnInfo{
 				CveContents: CveContents{
 					Ubuntu: {
-						Type:     Ubuntu,
-						Severity: "MEDIUM",
+						Type:          Ubuntu,
+						Cvss2Severity: "MEDIUM",
 					},
-					NVD: {
-						Type:       NVD,
-						Cvss2Score: 7.0,
+					NvdXML: {
+						Type:          NvdXML,
+						Cvss2Score:    7.0,
+						Cvss2Severity: "HIGH",
 					},
 				},
 			},
 			out: CveContentCvss{
-				Type: NVD,
+				Type: NvdXML,
 				Value: Cvss{
 					Type:     CVSS2,
 					Score:    7.0,
@@ -810,12 +811,13 @@ func TestMaxCvssScores(t *testing.T) {
 			in: VulnInfo{
 				CveContents: CveContents{
 					Ubuntu: {
-						Type:     Ubuntu,
-						Severity: "MEDIUM",
+						Type:          Ubuntu,
+						Cvss2Severity: "MEDIUM",
 					},
-					NVD: {
-						Type:       NVD,
-						Cvss2Score: 4.0,
+					NvdXML: {
+						Type:          NvdXML,
+						Cvss2Score:    4.0,
+						Cvss2Severity: "MEDIUM",
 					},
 				},
 				DistroAdvisories: []DistroAdvisory{
@@ -825,7 +827,7 @@ func TestMaxCvssScores(t *testing.T) {
 				},
 			},
 			out: CveContentCvss{
-				Type: NVD,
+				Type: NvdXML,
 				Value: Cvss{
 					Type:     CVSS2,
 					Score:    4,
@@ -861,18 +863,18 @@ func TestFormatMaxCvssScore(t *testing.T) {
 		{
 			in: VulnInfo{
 				CveContents: CveContents{
-					JVN: {
-						Type:       JVN,
-						Severity:   "HIGH",
-						Cvss2Score: 8.3,
+					Jvn: {
+						Type:          Jvn,
+						Cvss2Severity: "HIGH",
+						Cvss2Score:    8.3,
 					},
 					RedHat: {
-						Type:       RedHat,
-						Severity:   "HIGH",
-						Cvss3Score: 8.0,
+						Type:          RedHat,
+						Cvss2Severity: "HIGH",
+						Cvss3Score:    8.0,
 					},
-					NVD: {
-						Type:       NVD,
+					NvdXML: {
+						Type:       NvdXML,
 						Cvss2Score: 8.1,
 						// Severity is NIOT included in NVD
 					},
@@ -883,19 +885,20 @@ func TestFormatMaxCvssScore(t *testing.T) {
 		{
 			in: VulnInfo{
 				CveContents: CveContents{
-					JVN: {
-						Type:       JVN,
-						Severity:   "HIGH",
-						Cvss2Score: 8.3,
+					Jvn: {
+						Type:          Jvn,
+						Cvss2Severity: "HIGH",
+						Cvss2Score:    8.3,
 					},
 					RedHat: {
-						Type:       RedHat,
-						Severity:   "HIGH",
-						Cvss2Score: 8.0,
-						Cvss3Score: 9.9,
+						Type:          RedHat,
+						Cvss2Severity: "HIGH",
+						Cvss2Score:    8.0,
+						Cvss3Severity: "HIGH",
+						Cvss3Score:    9.9,
 					},
-					NVD: {
-						Type:       NVD,
+					NvdXML: {
+						Type:       NvdXML,
 						Cvss2Score: 8.1,
 					},
 				},
@@ -934,3 +937,100 @@ func TestSortPackageStatues(t *testing.T) {
 		}
 	}
 }
+
+func TestStorePackageStatueses(t *testing.T) {
+	var tests = []struct {
+		pkgstats PackageStatuses
+		in       PackageStatus
+		out      PackageStatuses
+	}{
+		{
+			pkgstats: PackageStatuses{
+				{Name: "a"},
+				{Name: "b"},
+			},
+			in: PackageStatus{
+				Name: "c",
+			},
+			out: PackageStatuses{
+				{Name: "a"},
+				{Name: "b"},
+				{Name: "c"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		out := tt.pkgstats.Store(tt.in)
+		if ok := reflect.DeepEqual(tt.out, out); !ok {
+			t.Errorf("\nexpected: %v\n  actual: %v\n", tt.out, out)
+		}
+	}
+}
+
+func TestAppendIfMissing(t *testing.T) {
+	var tests = []struct {
+		in  Confidences
+		arg Confidence
+		out Confidences
+	}{
+		{
+			in: Confidences{
+				CpeNameMatch,
+			},
+			arg: CpeNameMatch,
+			out: Confidences{
+				CpeNameMatch,
+			},
+		},
+		{
+			in: Confidences{
+				CpeNameMatch,
+			},
+			arg: ChangelogExactMatch,
+			out: Confidences{
+				CpeNameMatch,
+				ChangelogExactMatch,
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt.in.AppendIfMissing(tt.arg)
+		if !reflect.DeepEqual(tt.in, tt.out) {
+			t.Errorf("\nexpected: %v\n  actual: %v\n", tt.out, tt.in)
+		}
+	}
+}
+
+func TestSortByConfiden(t *testing.T) {
+	var tests = []struct {
+		in  Confidences
+		out Confidences
+	}{
+		{
+			in: Confidences{
+				OvalMatch,
+				CpeNameMatch,
+			},
+			out: Confidences{
+				OvalMatch,
+				CpeNameMatch,
+			},
+		},
+		{
+			in: Confidences{
+				CpeNameMatch,
+				OvalMatch,
+			},
+			out: Confidences{
+				OvalMatch,
+				CpeNameMatch,
+			},
+		},
+	}
+	for _, tt := range tests {
+		act := tt.in.SortByConfident()
+		if !reflect.DeepEqual(tt.out, act) {
+			t.Errorf("\nexpected: %v\n  actual: %v\n", tt.out, act)
+		}
+	}
+}

oval/alpine.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,7 +21,7 @@ import (
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/util"
-	ovalmodels "github.com/kotakanbe/goval-dictionary/models"
+	"github.com/kotakanbe/goval-dictionary/db"
 )
 
 // Alpine is the struct of Alpine Linux
@@ -39,44 +39,36 @@ func NewAlpine() Alpine {
 }
 
 // FillWithOval returns scan result after updating CVE info by OVAL
-func (o Alpine) FillWithOval(r *models.ScanResult) (err error) {
+func (o Alpine) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
 	var relatedDefs ovalResult
-	if o.isFetchViaHTTP() {
+	if config.Conf.OvalDict.IsFetchViaHTTP() {
 		if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil {
-			return err
+			return 0, err
 		}
 	} else {
-		if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil {
-			return err
+		if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil {
+			return 0, err
 		}
 	}
 	for _, defPacks := range relatedDefs.entries {
 		o.update(r, defPacks)
 	}
 
-	return nil
+	return len(relatedDefs.entries), nil
 }
 
 func (o Alpine) update(r *models.ScanResult, defPacks defPacks) {
-	ovalContent := *o.convertToModel(&defPacks.def)
 	cveID := defPacks.def.Advisory.Cves[0].CveID
 	vinfo, ok := r.ScannedCves[cveID]
 	if !ok {
 		util.Log.Debugf("%s is newly detected by OVAL", cveID)
 		vinfo = models.VulnInfo{
 			CveID:       cveID,
-			Confidence:  models.OvalMatch,
-			CveContents: models.NewCveContents(ovalContent),
+			Confidences: []models.Confidence{models.OvalMatch},
 		}
 	}
 
-	vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family)
+	vinfo.AffectedPackages = defPacks.toPackStatuses()
 	vinfo.AffectedPackages.Sort()
 	r.ScannedCves[cveID] = vinfo
 }
-
-func (o Alpine) convertToModel(def *ovalmodels.Definition) *models.CveContent {
-	return &models.CveContent{
-		CveID: def.Advisory.Cves[0].CveID,
-	}
-}

oval/debian.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@ import (
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/util"
+	"github.com/kotakanbe/goval-dictionary/db"
 	ovalmodels "github.com/kotakanbe/goval-dictionary/models"
 )
 
@@ -37,7 +38,7 @@ func (o DebianBase) update(r *models.ScanResult, defPacks defPacks) {
 		util.Log.Debugf("%s is newly detected by OVAL", defPacks.def.Debian.CveID)
 		vinfo = models.VulnInfo{
 			CveID:       defPacks.def.Debian.CveID,
-			Confidence:  models.OvalMatch,
+			Confidences: []models.Confidence{models.OvalMatch},
 			CveContents: models.NewCveContents(ovalContent),
 		}
 	} else {
@@ -51,20 +52,28 @@ func (o DebianBase) update(r *models.ScanResult, defPacks defPacks) {
 				defPacks.def.Debian.CveID)
 			cveContents = models.CveContents{}
 		}
-		if vinfo.Confidence.Score < models.OvalMatch.Score {
-			vinfo.Confidence = models.OvalMatch
-		}
+		vinfo.Confidences.AppendIfMissing(models.OvalMatch)
 		cveContents[ctype] = ovalContent
 		vinfo.CveContents = cveContents
 	}
 
 	// uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames)
 	for _, pack := range vinfo.AffectedPackages {
-		notFixedYet, _ := defPacks.actuallyAffectedPackNames[pack.Name]
-		defPacks.actuallyAffectedPackNames[pack.Name] = notFixedYet
+		defPacks.actuallyAffectedPackNames[pack.Name] = pack.NotFixedYet
 	}
 
-	vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family)
+	// update notFixedYet of SrcPackage
+	for binName := range defPacks.actuallyAffectedPackNames {
+		if srcPack, ok := r.SrcPackages.FindByBinName(binName); ok {
+			for _, p := range defPacks.def.AffectedPacks {
+				if p.Name == srcPack.Name {
+					defPacks.actuallyAffectedPackNames[binName] = p.NotFixedYet
+				}
+			}
+		}
+	}
+
+	vinfo.AffectedPackages = defPacks.toPackStatuses()
 	vinfo.AffectedPackages.Sort()
 	r.ScannedCves[defPacks.def.Debian.CveID] = vinfo
 }
@@ -80,11 +89,11 @@ func (o DebianBase) convertToModel(def *ovalmodels.Definition) *models.CveConten
 	}
 
 	return &models.CveContent{
-		CveID:      def.Debian.CveID,
-		Title:      def.Title,
-		Summary:    def.Description,
-		Severity:   def.Advisory.Severity,
-		References: refs,
+		CveID:         def.Debian.CveID,
+		Title:         def.Title,
+		Summary:       def.Description,
+		Cvss2Severity: def.Advisory.Severity,
+		References:    refs,
 	}
 }
 
@@ -105,17 +114,17 @@ func NewDebian() Debian {
 }
 
 // FillWithOval returns scan result after updating CVE info by OVAL
-func (o Debian) FillWithOval(r *models.ScanResult) (err error) {
+func (o Debian) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
 
 	//Debian's uname gives both of kernel release(uname -r), version(kernel-image version)
 	linuxImage := "linux-image-" + r.RunningKernel.Release
 
 	// Add linux and set the version of running kernel to search OVAL.
-	newVer := ""
-	if p, ok := r.Packages[linuxImage]; ok {
-		newVer = p.NewVersion
-	}
 	if r.Container.ContainerID == "" {
+		newVer := ""
+		if p, ok := r.Packages[linuxImage]; ok {
+			newVer = p.NewVersion
+		}
 		r.Packages["linux"] = models.Package{
 			Name:       "linux",
 			Version:    r.RunningKernel.Version,
@@ -124,13 +133,13 @@ func (o Debian) FillWithOval(r *models.ScanResult) (err error) {
 	}
 
 	var relatedDefs ovalResult
-	if o.isFetchViaHTTP() {
+	if config.Conf.OvalDict.IsFetchViaHTTP() {
 		if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil {
-			return err
+			return 0, err
 		}
 	} else {
-		if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil {
-			return err
+		if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil {
+			return 0, err
 		}
 	}
 
@@ -159,7 +168,7 @@ func (o Debian) FillWithOval(r *models.ScanResult) (err error) {
 			vuln.CveContents[models.Debian] = cont
 		}
 	}
-	return nil
+	return len(relatedDefs.entries), nil
 }
 
 // Ubuntu is the interface for Debian OVAL
@@ -179,7 +188,7 @@ func NewUbuntu() Ubuntu {
 }
 
 // FillWithOval returns scan result after updating CVE info by OVAL
-func (o Ubuntu) FillWithOval(r *models.ScanResult) (err error) {
+func (o Ubuntu) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
 	ovalKernelImageNames := []string{
 		"linux-aws",
 		"linux-azure",
@@ -234,13 +243,13 @@ func (o Ubuntu) FillWithOval(r *models.ScanResult) (err error) {
 	}
 
 	var relatedDefs ovalResult
-	if o.isFetchViaHTTP() {
+	if config.Conf.OvalDict.IsFetchViaHTTP() {
 		if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil {
-			return err
+			return 0, err
 		}
 	} else {
-		if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil {
-			return err
+		if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil {
+			return 0, err
 		}
 	}
 
@@ -271,5 +280,5 @@ func (o Ubuntu) FillWithOval(r *models.ScanResult) (err error) {
 			vuln.CveContents[models.Ubuntu] = cont
 		}
 	}
-	return nil
+	return len(relatedDefs.entries), nil
 }

oval/debian_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

oval/oval.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -23,22 +23,21 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/future-architect/vuls/config"
+	cnf "github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/util"
 	"github.com/kotakanbe/goval-dictionary/db"
-	ovallog "github.com/kotakanbe/goval-dictionary/log"
 	"github.com/parnurzeal/gorequest"
 )
 
 // Client is the interface of OVAL client.
 type Client interface {
 	CheckHTTPHealth() error
-	FillWithOval(r *models.ScanResult) error
+	FillWithOval(db.DB, *models.ScanResult) (int, error)
 
 	// CheckIfOvalFetched checks if oval entries are in DB by family, release.
-	CheckIfOvalFetched(string, string) (bool, error)
-	CheckIfOvalFresh(string, string) (bool, error)
+	CheckIfOvalFetched(db.DB, string, string) (bool, error)
+	CheckIfOvalFresh(db.DB, string, string) (bool, error)
 }
 
 // Base is a base struct
@@ -48,11 +47,11 @@ type Base struct {
 
 // CheckHTTPHealth do health check
 func (b Base) CheckHTTPHealth() error {
-	if !b.isFetchViaHTTP() {
+	if !cnf.Conf.OvalDict.IsFetchViaHTTP() {
 		return nil
 	}
 
-	url := fmt.Sprintf("%s/health", config.Conf.OvalDBURL)
+	url := fmt.Sprintf("%s/health", cnf.Conf.OvalDict.URL)
 	var errs []error
 	var resp *http.Response
 	resp, _, errs = gorequest.New().Get(url).End()
@@ -66,20 +65,9 @@ func (b Base) CheckHTTPHealth() error {
 }
 
 // CheckIfOvalFetched checks if oval entries are in DB by family, release.
-func (b Base) CheckIfOvalFetched(osFamily, release string) (fetched bool, err error) {
-	ovallog.Initialize(config.Conf.LogDir)
-	if !b.isFetchViaHTTP() {
-		var ovaldb db.DB
-		if ovaldb, err = db.NewDB(
-			osFamily,
-			config.Conf.OvalDBType,
-			config.Conf.OvalDBPath,
-			config.Conf.DebugSQL,
-		); err != nil {
-			return false, err
-		}
-		defer ovaldb.CloseDB()
-		count, err := ovaldb.CountDefs(osFamily, release)
+func (b Base) CheckIfOvalFetched(driver db.DB, osFamily, release string) (fetched bool, err error) {
+	if !cnf.Conf.OvalDict.IsFetchViaHTTP() {
+		count, err := driver.CountDefs(osFamily, release)
 		if err != nil {
 			return false, fmt.Errorf("Failed to count OVAL defs: %s, %s, %v",
 				osFamily, release, err)
@@ -87,7 +75,7 @@ func (b Base) CheckIfOvalFetched(osFamily, release string) (fetched bool, err er
 		return 0 < count, nil
 	}
 
-	url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "count", osFamily, release)
+	url, _ := util.URLPathJoin(cnf.Conf.OvalDict.URL, "count", osFamily, release)
 	resp, body, errs := gorequest.New().Get(url).End()
 	if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
 		return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
@@ -102,23 +90,12 @@ func (b Base) CheckIfOvalFetched(osFamily, release string) (fetched bool, err er
 }
 
 // CheckIfOvalFresh checks if oval entries are fresh enough
-func (b Base) CheckIfOvalFresh(osFamily, release string) (ok bool, err error) {
-	ovallog.Initialize(config.Conf.LogDir)
+func (b Base) CheckIfOvalFresh(driver db.DB, osFamily, release string) (ok bool, err error) {
 	var lastModified time.Time
-	if !b.isFetchViaHTTP() {
-		var ovaldb db.DB
-		if ovaldb, err = db.NewDB(
-			osFamily,
-			config.Conf.OvalDBType,
-			config.Conf.OvalDBPath,
-			config.Conf.DebugSQL,
-		); err != nil {
-			return false, err
-		}
-		defer ovaldb.CloseDB()
-		lastModified = ovaldb.GetLastModified(osFamily, release)
+	if !cnf.Conf.OvalDict.IsFetchViaHTTP() {
+		lastModified = driver.GetLastModified(osFamily, release)
 	} else {
-		url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "lastmodified", osFamily, release)
+		url, _ := util.URLPathJoin(cnf.Conf.OvalDict.URL, "lastmodified", osFamily, release)
 		resp, body, errs := gorequest.New().Get(url).End()
 		if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
 			return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
@@ -141,8 +118,3 @@ func (b Base) CheckIfOvalFresh(osFamily, release string) (ok bool, err error) {
 	util.Log.Infof("OVAL is fresh: %s %s ", osFamily, release)
 	return true, nil
 }
-
-func (b Base) isFetchViaHTTP() bool {
-	// Default value of OvalDBType is sqlite3
-	return config.Conf.OvalDBURL != "" && config.Conf.OvalDBType == "sqlite3"
-}

oval/redhat.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@ import (
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/util"
+	"github.com/kotakanbe/goval-dictionary/db"
 	ovalmodels "github.com/kotakanbe/goval-dictionary/models"
 )
 
@@ -34,20 +35,20 @@ type RedHatBase struct {
 }
 
 // FillWithOval returns scan result after updating CVE info by OVAL
-func (o RedHatBase) FillWithOval(r *models.ScanResult) (err error) {
+func (o RedHatBase) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
 	var relatedDefs ovalResult
-	if o.isFetchViaHTTP() {
+	if config.Conf.OvalDict.IsFetchViaHTTP() {
 		if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil {
-			return err
+			return 0, err
 		}
 	} else {
-		if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil {
-			return err
+		if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil {
+			return 0, err
 		}
 	}
 
 	for _, defPacks := range relatedDefs.entries {
-		o.update(r, defPacks)
+		nCVEs += o.update(r, defPacks)
 	}
 
 	for _, vuln := range r.ScannedCves {
@@ -64,7 +65,8 @@ func (o RedHatBase) FillWithOval(r *models.ScanResult) (err error) {
 			}
 		}
 	}
-	return nil
+
+	return nCVEs, nil
 }
 
 var kernelRelatedPackNames = map[string]bool{
@@ -94,9 +96,11 @@ var kernelRelatedPackNames = map[string]bool{
 	"kernel-tools":            true,
 	"kernel-tools-libs":       true,
 	"kernel-tools-libs-devel": true,
+	"perf":                    true,
+	"python-perf":             true,
 }
 
-func (o RedHatBase) update(r *models.ScanResult, defPacks defPacks) {
+func (o RedHatBase) update(r *models.ScanResult, defPacks defPacks) (nCVEs int) {
 	ctype := models.NewCveContentType(o.family)
 	for _, cve := range defPacks.def.Advisory.Cves {
 		ovalContent := *o.convertToModel(cve.CveID, &defPacks.def)
@@ -105,34 +109,43 @@ func (o RedHatBase) update(r *models.ScanResult, defPacks defPacks) {
 			util.Log.Debugf("%s is newly detected by OVAL", cve.CveID)
 			vinfo = models.VulnInfo{
 				CveID:       cve.CveID,
-				Confidence:  models.OvalMatch,
+				Confidences: models.Confidences{models.OvalMatch},
 				CveContents: models.NewCveContents(ovalContent),
 			}
+			nCVEs++
 		} else {
 			cveContents := vinfo.CveContents
-			if _, ok := vinfo.CveContents[ctype]; ok {
-				util.Log.Debugf("%s OVAL will be overwritten", cve.CveID)
+			if v, ok := vinfo.CveContents[ctype]; ok {
+				if v.LastModified.After(ovalContent.LastModified) {
+					util.Log.Debugf("%s, OvalID: %d ignroed: ",
+						cve.CveID, defPacks.def.ID)
+					continue
+				} else {
+					util.Log.Debugf("%s OVAL will be overwritten", cve.CveID)
+				}
 			} else {
 				util.Log.Debugf("%s also detected by OVAL", cve.CveID)
 				cveContents = models.CveContents{}
 			}
 
-			if vinfo.Confidence.Score < models.OvalMatch.Score {
-				vinfo.Confidence = models.OvalMatch
-			}
+			vinfo.Confidences.AppendIfMissing(models.OvalMatch)
 			cveContents[ctype] = ovalContent
 			vinfo.CveContents = cveContents
 		}
 
 		// uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames)
 		for _, pack := range vinfo.AffectedPackages {
-			notFixedYet, _ := defPacks.actuallyAffectedPackNames[pack.Name]
-			defPacks.actuallyAffectedPackNames[pack.Name] = notFixedYet
+			if nfy, ok := defPacks.actuallyAffectedPackNames[pack.Name]; !ok {
+				defPacks.actuallyAffectedPackNames[pack.Name] = pack.NotFixedYet
+			} else if nfy {
+				defPacks.actuallyAffectedPackNames[pack.Name] = true
+			}
 		}
-		vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family)
+		vinfo.AffectedPackages = defPacks.toPackStatuses()
 		vinfo.AffectedPackages.Sort()
 		r.ScannedCves[cve.CveID] = vinfo
 	}
+	return
 }
 
 func (o RedHatBase) convertToModel(cveID string, def *ovalmodels.Definition) *models.CveContent {
@@ -157,20 +170,32 @@ func (o RedHatBase) convertToModel(cveID string, def *ovalmodels.Definition) *mo
 			severity = cve.Impact
 		}
 
+		sev2, sev3 := "", ""
+		if score2 != 0 {
+			sev2 = severity
+		}
+		if score3 != 0 {
+			sev3 = severity
+		}
+
+		// CWE-ID in RedHat OVAL may have multiple cweIDs separated by space
+		cwes := strings.Fields(cve.Cwe)
+
 		return &models.CveContent{
-			Type:         models.NewCveContentType(o.family),
-			CveID:        cve.CveID,
-			Title:        def.Title,
-			Summary:      def.Description,
-			Severity:     severity,
-			Cvss2Score:   score2,
-			Cvss2Vector:  vec2,
-			Cvss3Score:   score3,
-			Cvss3Vector:  vec3,
-			References:   refs,
-			CweID:        cve.Cwe,
-			Published:    def.Advisory.Issued,
-			LastModified: def.Advisory.Updated,
+			Type:          models.NewCveContentType(o.family),
+			CveID:         cve.CveID,
+			Title:         def.Title,
+			Summary:       def.Description,
+			Cvss2Score:    score2,
+			Cvss2Vector:   vec2,
+			Cvss2Severity: sev2,
+			Cvss3Score:    score3,
+			Cvss3Vector:   vec3,
+			Cvss3Severity: sev3,
+			References:    refs,
+			CweIDs:        cwes,
+			Published:     def.Advisory.Issued,
+			LastModified:  def.Advisory.Updated,
 		}
 	}
 	return nil
@@ -199,7 +224,7 @@ func (o RedHatBase) parseCvss3(scoreVector string) (score float64, vector string
 		if score, err = strconv.ParseFloat(ss[0], 64); err != nil {
 			return 0, ""
 		}
-		return score, strings.Join(ss[1:], "/")
+		return score, fmt.Sprintf("CVSS:3.0/%s", ss[1])
 	}
 	return 0, ""
 }
@@ -236,7 +261,7 @@ func NewCentOS() CentOS {
 	}
 }
 
-// Oracle is the interface for CentOS OVAL
+// Oracle is the interface for Oracle OVAL
 type Oracle struct {
 	RedHatBase
 }

oval/redhat_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -72,7 +72,7 @@ func TestParseCvss3(t *testing.T) {
 			in: "5.6/CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
 			out: out{
 				score:  5.6,
-				vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
+				vector: "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L",
 			},
 		},
 		{

oval/suse.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@ import (
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/util"
+	"github.com/kotakanbe/goval-dictionary/db"
 	ovalmodels "github.com/kotakanbe/goval-dictionary/models"
 )
 
@@ -40,15 +41,15 @@ func NewSUSE() SUSE {
 }
 
 // FillWithOval returns scan result after updating CVE info by OVAL
-func (o SUSE) FillWithOval(r *models.ScanResult) (err error) {
+func (o SUSE) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err error) {
 	var relatedDefs ovalResult
-	if o.isFetchViaHTTP() {
+	if config.Conf.OvalDict.IsFetchViaHTTP() {
 		if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil {
-			return err
+			return 0, err
 		}
 	} else {
-		if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil {
-			return err
+		if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil {
+			return 0, err
 		}
 	}
 	for _, defPacks := range relatedDefs.entries {
@@ -61,7 +62,7 @@ func (o SUSE) FillWithOval(r *models.ScanResult) (err error) {
 			vuln.CveContents[models.SUSE] = cont
 		}
 	}
-	return nil
+	return len(relatedDefs.entries), nil
 }
 
 func (o SUSE) update(r *models.ScanResult, defPacks defPacks) {
@@ -72,7 +73,7 @@ func (o SUSE) update(r *models.ScanResult, defPacks defPacks) {
 		util.Log.Debugf("%s is newly detected by OVAL", defPacks.def.Title)
 		vinfo = models.VulnInfo{
 			CveID:       defPacks.def.Title,
-			Confidence:  models.OvalMatch,
+			Confidences: models.Confidences{models.OvalMatch},
 			CveContents: models.NewCveContents(ovalContent),
 		}
 	} else {
@@ -84,19 +85,16 @@ func (o SUSE) update(r *models.ScanResult, defPacks defPacks) {
 			util.Log.Debugf("%s is also detected by OVAL", defPacks.def.Title)
 			cveContents = models.CveContents{}
 		}
-		if vinfo.Confidence.Score < models.OvalMatch.Score {
-			vinfo.Confidence = models.OvalMatch
-		}
+		vinfo.Confidences.AppendIfMissing(models.OvalMatch)
 		cveContents[ctype] = ovalContent
 		vinfo.CveContents = cveContents
 	}
 
 	// uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames)
 	for _, pack := range vinfo.AffectedPackages {
-		notFixedYet, _ := defPacks.actuallyAffectedPackNames[pack.Name]
-		defPacks.actuallyAffectedPackNames[pack.Name] = notFixedYet
+		defPacks.actuallyAffectedPackNames[pack.Name] = pack.NotFixedYet
 	}
-	vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family)
+	vinfo.AffectedPackages = defPacks.toPackStatuses()
 	vinfo.AffectedPackages.Sort()
 	r.ScannedCves[defPacks.def.Title] = vinfo
 }

oval/util.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -32,7 +32,6 @@ import (
 	debver "github.com/knqyf263/go-deb-version"
 	rpmver "github.com/knqyf263/go-rpm-version"
 	"github.com/kotakanbe/goval-dictionary/db"
-	ovallog "github.com/kotakanbe/goval-dictionary/log"
 	ovalmodels "github.com/kotakanbe/goval-dictionary/models"
 	"github.com/parnurzeal/gorequest"
 )
@@ -48,7 +47,7 @@ type defPacks struct {
 	actuallyAffectedPackNames map[string]bool
 }
 
-func (e defPacks) toPackStatuses(family string) (ps models.PackageStatuses) {
+func (e defPacks) toPackStatuses() (ps models.PackageStatuses) {
 	for name, notFixedYet := range e.actuallyAffectedPackNames {
 		ps = append(ps, models.PackageStatus{
 			Name:        name,
@@ -69,7 +68,7 @@ func (e *ovalResult) upsert(def ovalmodels.Definition, packName string, notFixed
 		}
 	}
 	e.entries = append(e.entries, defPacks{
-		def: def,
+		def:                       def,
 		actuallyAffectedPackNames: map[string]bool{packName: notFixedYet},
 	})
 
@@ -109,15 +108,14 @@ func getDefsByPackNameViaHTTP(r *models.ScanResult) (
 				NewVersionRelease: pack.FormatVer(),
 				isSrcPack:         false,
 			}
-			for _, pack := range r.SrcPackages {
-				reqChan <- request{
-					packName:        pack.Name,
-					binaryPackNames: pack.BinaryNames,
-					versionRelease:  pack.Version,
-					isSrcPack:       true,
-				}
+		}
+		for _, pack := range r.SrcPackages {
+			reqChan <- request{
+				packName:        pack.Name,
+				binaryPackNames: pack.BinaryNames,
+				versionRelease:  pack.Version,
+				isSrcPack:       true,
 			}
-
 		}
 	}()
 
@@ -128,7 +126,7 @@ func getDefsByPackNameViaHTTP(r *models.ScanResult) (
 			select {
 			case req := <-reqChan:
 				url, err := util.URLPathJoin(
-					config.Conf.OvalDBURL,
+					config.Conf.OvalDict.URL,
 					"packs",
 					r.Family,
 					r.Release,
@@ -218,21 +216,7 @@ func httpGet(url string, req request, resChan chan<- response, errChan chan<- er
 	}
 }
 
-func getDefsByPackNameFromOvalDB(r *models.ScanResult) (relatedDefs ovalResult, err error) {
-	ovallog.Initialize(config.Conf.LogDir)
-	path := config.Conf.OvalDBURL
-	if config.Conf.OvalDBType == "sqlite3" {
-		path = config.Conf.OvalDBPath
-	}
-	util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path)
-
-	var ovaldb db.DB
-	if ovaldb, err = db.NewDB(r.Family, config.Conf.OvalDBType,
-		path, config.Conf.DebugSQL); err != nil {
-		return
-	}
-	defer ovaldb.CloseDB()
-
+func getDefsByPackNameFromOvalDB(driver db.DB, r *models.ScanResult) (relatedDefs ovalResult, err error) {
 	requests := []request{}
 	for _, pack := range r.Packages {
 		requests = append(requests, request{
@@ -252,9 +236,9 @@ func getDefsByPackNameFromOvalDB(r *models.ScanResult) (relatedDefs ovalResult,
 	}
 
 	for _, req := range requests {
-		definitions, err := ovaldb.GetByPackName(r.Release, req.packName)
+		definitions, err := driver.GetByPackName(r.Release, req.packName)
 		if err != nil {
-			return relatedDefs, fmt.Errorf("Failed to get %s OVAL info by package name: %v", r.Family, err)
+			return relatedDefs, fmt.Errorf("Failed to get %s OVAL info by package: %#v, err: %s", r.Family, req, err)
 		}
 		for _, def := range definitions {
 			affected, notFixedYet := isOvalDefAffected(def, req, r.Family, r.RunningKernel)
@@ -319,8 +303,12 @@ func isOvalDefAffected(def ovalmodels.Definition, req request, family string, ru
 				// Unable to judge whether fixed or not fixed of src package(Ubuntu, Debian)
 				return true, false
 			}
+
+			// `offline` or `fast` scan mode can't get a updatable version.
+			// In these mode, the blow field was set empty.
+			// Vuls can not judge fixed or unfixed.
 			if req.NewVersionRelease == "" {
-				return true, true
+				return true, false
 			}
 
 			// compare version: newVer vs oval

oval/util_test.go

@@ -105,9 +105,8 @@ func TestUpsert(t *testing.T) {
 
 func TestDefpacksToPackStatuses(t *testing.T) {
 	type in struct {
-		dp     defPacks
-		family string
-		packs  models.Packages
+		dp    defPacks
+		packs models.Packages
 	}
 	var tests = []struct {
 		in  in
@@ -116,7 +115,6 @@ func TestDefpacksToPackStatuses(t *testing.T) {
 		// Ubuntu
 		{
 			in: in{
-				family: "ubuntu",
 				dp: defPacks{
 					def: ovalmodels.Definition{
 						AffectedPacks: []ovalmodels.Package{
@@ -154,7 +152,7 @@ func TestDefpacksToPackStatuses(t *testing.T) {
 		},
 	}
 	for i, tt := range tests {
-		actual := tt.in.dp.toPackStatuses(tt.in.family)
+		actual := tt.in.dp.toPackStatuses()
 		sort.Slice(actual, func(i, j int) bool {
 			return actual[i].Name < actual[j].Name
 		})
@@ -321,7 +319,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    true,
 			notFixedYet: false,
 		},
-		// RedHat
+		// 5 RedHat
 		{
 			in: in{
 				family: "redhat",
@@ -339,64 +337,16 @@ func TestIsOvalDefAffected(t *testing.T) {
 					},
 				},
 				req: request{
-					packName:       "b",
-					isSrcPack:      false,
-					versionRelease: "0:1.2.3-45.el6_7.7",
+					packName:          "b",
+					isSrcPack:         false,
+					versionRelease:    "0:1.2.3-45.el6_7.7",
+					NewVersionRelease: "",
 				},
 			},
 			affected:    true,
-			notFixedYet: true,
-		},
-		{
-			in: in{
-				family: "redhat",
-				def: ovalmodels.Definition{
-					AffectedPacks: []ovalmodels.Package{
-						{
-							Name:        "a",
-							NotFixedYet: false,
-						},
-						{
-							Name:        "b",
-							NotFixedYet: false,
-							Version:     "0:1.2.3-45.el6_7.8",
-						},
-					},
-				},
-				req: request{
-					packName:       "b",
-					isSrcPack:      false,
-					versionRelease: "0:1.2.3-45.el6_7.8",
-				},
-			},
-			affected:    false,
-			notFixedYet: false,
-		},
-		{
-			in: in{
-				family: "redhat",
-				def: ovalmodels.Definition{
-					AffectedPacks: []ovalmodels.Package{
-						{
-							Name:        "a",
-							NotFixedYet: false,
-						},
-						{
-							Name:        "b",
-							NotFixedYet: false,
-							Version:     "0:1.2.3-45.el6_7.8",
-						},
-					},
-				},
-				req: request{
-					packName:       "b",
-					isSrcPack:      false,
-					versionRelease: "0:1.2.3-45.el6_7.9",
-				},
-			},
-			affected:    false,
 			notFixedYet: false,
 		},
+		// 6 RedHat
 		{
 			in: in{
 				family: "redhat",
@@ -423,6 +373,86 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    true,
 			notFixedYet: true,
 		},
+		// 7 RedHat
+		{
+			in: in{
+				family: "redhat",
+				def: ovalmodels.Definition{
+					AffectedPacks: []ovalmodels.Package{
+						{
+							Name:        "a",
+							NotFixedYet: false,
+						},
+						{
+							Name:        "b",
+							NotFixedYet: false,
+							Version:     "0:1.2.3-45.el6_7.8",
+						},
+					},
+				},
+				req: request{
+					packName:       "b",
+					isSrcPack:      false,
+					versionRelease: "0:1.2.3-45.el6_7.8",
+				},
+			},
+			affected:    false,
+			notFixedYet: false,
+		},
+		// 8 RedHat
+		{
+			in: in{
+				family: "redhat",
+				def: ovalmodels.Definition{
+					AffectedPacks: []ovalmodels.Package{
+						{
+							Name:        "a",
+							NotFixedYet: false,
+						},
+						{
+							Name:        "b",
+							NotFixedYet: false,
+							Version:     "0:1.2.3-45.el6_7.8",
+						},
+					},
+				},
+				req: request{
+					packName:       "b",
+					isSrcPack:      false,
+					versionRelease: "0:1.2.3-45.el6_7.9",
+				},
+			},
+			affected:    false,
+			notFixedYet: false,
+		},
+		// 9 RedHat
+		{
+			in: in{
+				family: "redhat",
+				def: ovalmodels.Definition{
+					AffectedPacks: []ovalmodels.Package{
+						{
+							Name:        "a",
+							NotFixedYet: false,
+						},
+						{
+							Name:        "b",
+							NotFixedYet: false,
+							Version:     "0:1.2.3-45.el6_7.8",
+						},
+					},
+				},
+				req: request{
+					packName:          "b",
+					isSrcPack:         false,
+					versionRelease:    "0:1.2.3-45.el6_7.6",
+					NewVersionRelease: "0:1.2.3-45.el6_7.7",
+				},
+			},
+			affected:    true,
+			notFixedYet: true,
+		},
+		// 10 RedHat
 		{
 			in: in{
 				family: "redhat",
@@ -449,6 +479,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    true,
 			notFixedYet: false,
 		},
+		// 11 RedHat
 		{
 			in: in{
 				family: "redhat",
@@ -458,8 +489,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 							Name:        "a",
 							NotFixedYet: false,
 						},
-						{
-							Name:        "b",
+						{Name: "b",
 							NotFixedYet: false,
 							Version:     "0:1.2.3-45.el6_7.8",
 						},
@@ -475,6 +505,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    true,
 			notFixedYet: false,
 		},
+		// 12 RedHat
 		{
 			in: in{
 				family: "redhat",
@@ -500,6 +531,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    false,
 			notFixedYet: false,
 		},
+		// 13 RedHat
 		{
 			in: in{
 				family: "redhat",
@@ -525,7 +557,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    false,
 			notFixedYet: false,
 		},
-		// CentOS
+		// 14 CentOS
 		{
 			in: in{
 				family: "centos",
@@ -543,14 +575,16 @@ func TestIsOvalDefAffected(t *testing.T) {
 					},
 				},
 				req: request{
-					packName:       "b",
-					isSrcPack:      false,
-					versionRelease: "0:1.2.3-45.el6.centos.7",
+					packName:          "b",
+					isSrcPack:         false,
+					versionRelease:    "0:1.2.3-45.el6.centos.7",
+					NewVersionRelease: "",
 				},
 			},
 			affected:    true,
-			notFixedYet: true,
+			notFixedYet: false,
 		},
+		// 15
 		{
 			in: in{
 				family: "centos",
@@ -576,6 +610,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    false,
 			notFixedYet: false,
 		},
+		// 16
 		{
 			in: in{
 				family: "centos",
@@ -601,6 +636,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    false,
 			notFixedYet: false,
 		},
+		// 17
 		{
 			in: in{
 				family: "centos",
@@ -627,6 +663,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    true,
 			notFixedYet: true,
 		},
+		// 18
 		{
 			in: in{
 				family: "centos",
@@ -653,6 +690,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    true,
 			notFixedYet: false,
 		},
+		// 19
 		{
 			in: in{
 				family: "centos",
@@ -679,6 +717,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    true,
 			notFixedYet: false,
 		},
+		// 20
 		{
 			in: in{
 				family: "centos",
@@ -704,6 +743,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    false,
 			notFixedYet: false,
 		},
+		// 21
 		{
 			in: in{
 				family: "centos",
@@ -729,7 +769,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 			affected:    false,
 			notFixedYet: false,
 		},
-		// TODO: If vuls support Scientific, replace "centos" below to "scientific".
+		// 22
 		{
 			in: in{
 				family: "centos",
@@ -753,7 +793,7 @@ func TestIsOvalDefAffected(t *testing.T) {
 				},
 			},
 			affected:    true,
-			notFixedYet: true,
+			notFixedYet: false,
 		},
 		{
 			in: in{

report/azureblob.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -67,9 +67,9 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) {
 			}
 		}
 
-		if c.Conf.FormatShortText {
+		if c.Conf.FormatList {
 			k := key + "_short.txt"
-			b := []byte(formatShortPlainText(r))
+			b := []byte(formatList(r))
 			if err := createBlockBlob(cli, k, b); err != nil {
 				return err
 			}
@@ -111,19 +111,19 @@ func CheckIfAzureContainerExists() error {
 
 	found := false
 	for _, con := range r.Containers {
-		if con.Name == c.Conf.AzureContainer {
+		if con.Name == c.Conf.Azure.ContainerName {
 			found = true
 			break
 		}
 	}
 	if !found {
-		return fmt.Errorf("Container not found. Container: %s", c.Conf.AzureContainer)
+		return fmt.Errorf("Container not found. Container: %s", c.Conf.Azure.ContainerName)
 	}
 	return nil
 }
 
 func getBlobClient() (storage.BlobStorageClient, error) {
-	api, err := storage.NewBasicClient(c.Conf.AzureAccount, c.Conf.AzureKey)
+	api, err := storage.NewBasicClient(c.Conf.Azure.AccountName, c.Conf.Azure.AccountKey)
 	if err != nil {
 		return storage.BlobStorageClient{}, err
 	}
@@ -136,14 +136,14 @@ func createBlockBlob(cli storage.BlobStorageClient, k string, b []byte) error {
 		if b, err = gz(b); err != nil {
 			return err
 		}
-		k = k + ".gz"
+		k += ".gz"
 	}
 
-	ref := cli.GetContainerReference(c.Conf.AzureContainer)
+	ref := cli.GetContainerReference(c.Conf.Azure.ContainerName)
 	blob := ref.GetBlobReference(k)
 	if err := blob.CreateBlockBlobFromReader(bytes.NewReader(b), nil); err != nil {
 		return fmt.Errorf("Failed to upload data to %s/%s, %s",
-			c.Conf.AzureContainer, k, err)
+			c.Conf.Azure.ContainerName, k, err)
 	}
 	return nil
 }

report/chatwork.go

@@ -0,0 +1,73 @@
+package report
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+)
+
+// ChatWorkWriter send report to ChatWork
+type ChatWorkWriter struct{}
+
+func (w ChatWorkWriter) Write(rs ...models.ScanResult) (err error) {
+	conf := config.Conf.ChatWork
+
+	for _, r := range rs {
+		serverInfo := fmt.Sprintf("%s", r.ServerInfo())
+		if err = chatWorkpostMessage(conf.Room, conf.APIToken, serverInfo); err != nil {
+			return err
+		}
+
+		for _, vinfo := range r.ScannedCves {
+			maxCvss := vinfo.MaxCvssScore()
+			severity := strings.ToUpper(maxCvss.Value.Severity)
+			if severity == "" {
+				severity = "?"
+			}
+
+			message := fmt.Sprintf(`%s[info][title]"https://nvd.nist.gov/vuln/detail/%s" %s %s[/title]%s[/info]`,
+				serverInfo,
+				vinfo.CveID,
+				strconv.FormatFloat(maxCvss.Value.Score, 'f', 1, 64),
+				severity,
+				vinfo.Summaries(config.Conf.Lang, r.Family)[0].Value)
+
+			if err = chatWorkpostMessage(conf.Room, conf.APIToken, message); err != nil {
+				return err
+			}
+		}
+
+	}
+	return nil
+}
+
+func chatWorkpostMessage(room, token, message string) error {
+	uri := fmt.Sprintf("https://api.chatwork.com/v2/rooms/%s/messages=%s", room, token)
+
+	payload := url.Values{
+		"body": {message},
+	}
+
+	reqs, err := http.NewRequest("POST", uri, strings.NewReader(payload.Encode()))
+
+	reqs.Header.Add("X-ChatWorkToken", token)
+	reqs.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+	if err != nil {
+		return err
+	}
+	client := &http.Client{}
+
+	resp, err := client.Do(reqs)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	return nil
+}

report/chatwork_test.go

@@ -0,0 +1 @@
+package report

report/cve_client.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -28,10 +28,8 @@ import (
 
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/util"
-	cveconfig "github.com/kotakanbe/go-cve-dictionary/config"
 	cvedb "github.com/kotakanbe/go-cve-dictionary/db"
 	cve "github.com/kotakanbe/go-cve-dictionary/models"
-	log "github.com/sirupsen/logrus"
 )
 
 // CveClient is api client of CVE disctionary service.
@@ -43,12 +41,12 @@ type cvedictClient struct {
 }
 
 func (api *cvedictClient) initialize() {
-	api.baseURL = config.Conf.CveDBURL
+	api.baseURL = config.Conf.CveDict.URL
 }
 
 func (api cvedictClient) CheckHealth() error {
-	if !api.isFetchViaHTTP() {
-		util.Log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType)
+	if !config.Conf.CveDict.IsFetchViaHTTP() {
+		util.Log.Debugf("get cve-dictionary from %s", config.Conf.CveDict.Type)
 		return nil
 	}
 
@@ -70,12 +68,25 @@ type response struct {
 	CveDetail cve.CveDetail
 }
 
-func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails []*cve.CveDetail, err error) {
-	if !api.isFetchViaHTTP() {
-		return api.FetchCveDetailsFromCveDB(cveIDs)
+func (api cvedictClient) FetchCveDetails(driver cvedb.DB, cveIDs []string) (cveDetails []cve.CveDetail, err error) {
+	if !config.Conf.CveDict.IsFetchViaHTTP() {
+		for _, cveID := range cveIDs {
+			cveDetail, err := driver.Get(cveID)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to fetch CVE. err: %s", err)
+			}
+			if len(cveDetail.CveID) == 0 {
+				cveDetails = append(cveDetails, cve.CveDetail{
+					CveID: cveID,
+				})
+			} else {
+				cveDetails = append(cveDetails, *cveDetail)
+			}
+		}
+		return
 	}
 
-	api.baseURL = config.Conf.CveDBURL
+	api.baseURL = config.Conf.CveDict.URL
 	reqChan := make(chan string, len(cveIDs))
 	resChan := make(chan response, len(cveIDs))
 	errChan := make(chan error, len(cveIDs))
@@ -112,54 +123,25 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails []*cve.Cve
 		select {
 		case res := <-resChan:
 			if len(res.CveDetail.CveID) == 0 {
-				cveDetails = append(cveDetails, &cve.CveDetail{
+				cveDetails = append(cveDetails, cve.CveDetail{
 					CveID: res.Key,
 				})
 			} else {
-				cveDetails = append(cveDetails, &res.CveDetail)
+				cveDetails = append(cveDetails, res.CveDetail)
 			}
 		case err := <-errChan:
 			errs = append(errs, err)
 		case <-timeout:
-			return []*cve.CveDetail{}, fmt.Errorf("Timeout Fetching CVE")
+			return nil, fmt.Errorf("Timeout Fetching CVE")
 		}
 	}
 	if len(errs) != 0 {
-		return []*cve.CveDetail{},
+		return nil,
 			fmt.Errorf("Failed to fetch CVE. err: %v", errs)
 	}
 	return
 }
 
-func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails []*cve.CveDetail, err error) {
-	util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
-	cveconfig.Conf.DBType = config.Conf.CveDBType
-	if config.Conf.CveDBType == "sqlite3" {
-		cveconfig.Conf.DBPath = config.Conf.CveDBPath
-	} else {
-		cveconfig.Conf.DBPath = config.Conf.CveDBURL
-	}
-	cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
-
-	var driver cvedb.DB
-	if driver, err = cvedb.NewDB(cveconfig.Conf.DBType, cveconfig.Conf.DBPath, cveconfig.Conf.DebugSQL); err != nil {
-		log.Error(err)
-		return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err)
-	}
-
-	for _, cveID := range cveIDs {
-		cveDetail := driver.Get(cveID)
-		if len(cveDetail.CveID) == 0 {
-			cveDetails = append(cveDetails, &cve.CveDetail{
-				CveID: cveID,
-			})
-		} else {
-			cveDetails = append(cveDetails, cveDetail)
-		}
-	}
-	return
-}
-
 func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errChan chan<- error) {
 	var body string
 	var errs []error
@@ -194,36 +176,22 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh
 	}
 }
 
-type responseGetCveDetailByCpeName struct {
-	CpeName    string
-	CveDetails []cve.CveDetail
-}
-
-func (api cvedictClient) isFetchViaHTTP() bool {
-	// Default value of CveDBType is sqlite3
-	if config.Conf.CveDBURL != "" && config.Conf.CveDBType == "sqlite3" {
-		return true
-	}
-	return false
-}
-
-func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]*cve.CveDetail, error) {
-	if api.isFetchViaHTTP() {
-		api.baseURL = config.Conf.CveDBURL
+func (api cvedictClient) FetchCveDetailsByCpeName(driver cvedb.DB, cpeName string) ([]cve.CveDetail, error) {
+	if config.Conf.CveDict.IsFetchViaHTTP() {
+		api.baseURL = config.Conf.CveDict.URL
 		url, err := util.URLPathJoin(api.baseURL, "cpes")
 		if err != nil {
-			return []*cve.CveDetail{}, err
+			return nil, err
 		}
 
 		query := map[string]string{"name": cpeName}
 		util.Log.Debugf("HTTP Request to %s, query: %#v", url, query)
 		return api.httpPost(cpeName, url, query)
 	}
-
-	return api.FetchCveDetailsByCpeNameFromDB(cpeName)
+	return driver.GetByCpeURI(cpeName)
 }
 
-func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]*cve.CveDetail, error) {
+func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cve.CveDetail, error) {
 	var body string
 	var errs []error
 	var resp *http.Response
@@ -244,33 +212,13 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]*
 	}
 	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
 	if err != nil {
-		return []*cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err)
+		return nil, fmt.Errorf("HTTP Error %s", err)
 	}
 
-	cveDetails := []*cve.CveDetail{}
+	cveDetails := []cve.CveDetail{}
 	if err := json.Unmarshal([]byte(body), &cveDetails); err != nil {
-		return []*cve.CveDetail{},
+		return nil,
 			fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
 	}
 	return cveDetails, nil
 }
-
-func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) (cveDetails []*cve.CveDetail, err error) {
-	util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
-	cveconfig.Conf.DBType = config.Conf.CveDBType
-	if config.Conf.CveDBType == "sqlite3" {
-		cveconfig.Conf.DBPath = config.Conf.CveDBPath
-	} else {
-		cveconfig.Conf.DBPath = config.Conf.CveDBURL
-	}
-	cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
-
-	var driver cvedb.DB
-	if driver, err = cvedb.NewDB(cveconfig.Conf.DBType, cveconfig.Conf.DBPath, cveconfig.Conf.DebugSQL); err != nil {
-		log.Error(err)
-		return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err)
-	}
-
-	util.Log.Debugf("Opening DB (%s).", driver.Name())
-	return driver.GetByCpeName(cpeName), nil
-}

report/db_client.go

@@ -0,0 +1,188 @@
+package report
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/util"
+	gostdb "github.com/knqyf263/gost/db"
+	cvedb "github.com/kotakanbe/go-cve-dictionary/db"
+	ovaldb "github.com/kotakanbe/goval-dictionary/db"
+	exploitdb "github.com/mozqnet/go-exploitdb/db"
+)
+
+// DBClient is a dictionarie's db client for reporting
+type DBClient struct {
+	CveDB     cvedb.DB
+	OvalDB    ovaldb.DB
+	GostDB    gostdb.DB
+	ExploitDB exploitdb.DB
+}
+
+// DBClientConf has a configuration of Vulnerability DBs
+type DBClientConf struct {
+	CveDictCnf  config.GoCveDictConf
+	OvalDictCnf config.GovalDictConf
+	GostCnf     config.GostConf
+	ExploitCnf  config.ExploitConf
+	DebugSQL    bool
+}
+
+// NewDBClient returns db clients
+func NewDBClient(cnf DBClientConf) (dbclient *DBClient, locked bool, err error) {
+	cveDriver, locked, err := NewCveDB(cnf)
+	if locked {
+		return nil, true, fmt.Errorf("CveDB is locked: %s",
+			cnf.OvalDictCnf.SQLite3Path)
+	} else if err != nil {
+		return nil, locked, err
+	}
+
+	ovaldb, locked, err := NewOvalDB(cnf)
+	if locked {
+		return nil, true, fmt.Errorf("OvalDB is locked: %s",
+			cnf.OvalDictCnf.SQLite3Path)
+	} else if err != nil {
+		util.Log.Warnf("Unable to use OvalDB: %s, err: %s",
+			cnf.OvalDictCnf.SQLite3Path, err)
+	}
+
+	gostdb, locked, err := NewGostDB(cnf)
+	if locked {
+		return nil, true, fmt.Errorf("gostDB is locked: %s",
+			cnf.GostCnf.SQLite3Path)
+	} else if err != nil {
+		util.Log.Warnf("Unable to use gostDB: %s, err: %s",
+			cnf.GostCnf.SQLite3Path, err)
+	}
+
+	exploitdb, locked, err := NewExploitDB(cnf)
+	if locked {
+		return nil, true, fmt.Errorf("exploitDB is locked: %s",
+			cnf.ExploitCnf.SQLite3Path)
+	} else if err != nil {
+		util.Log.Warnf("Unable to use exploitDB: %s, err: %s",
+			cnf.ExploitCnf.SQLite3Path, err)
+	}
+
+	return &DBClient{
+		CveDB:     cveDriver,
+		OvalDB:    ovaldb,
+		GostDB:    gostdb,
+		ExploitDB: exploitdb,
+	}, false, nil
+}
+
+// NewCveDB returns cve db client
+func NewCveDB(cnf DBClientConf) (driver cvedb.DB, locked bool, err error) {
+	if config.Conf.CveDict.IsFetchViaHTTP() {
+		return nil, false, nil
+	}
+	util.Log.Debugf("open cve-dictionary db (%s)", cnf.CveDictCnf.Type)
+	path := cnf.CveDictCnf.URL
+	if cnf.CveDictCnf.Type == "sqlite3" {
+		path = cnf.CveDictCnf.SQLite3Path
+	}
+
+	util.Log.Debugf("Open cve-dictionary db (%s): %s", cnf.CveDictCnf.Type, path)
+	driver, locked, err = cvedb.NewDB(cnf.CveDictCnf.Type, path, cnf.DebugSQL)
+	if err != nil {
+		err = fmt.Errorf("Failed to init CVE DB. err: %s, path: %s", err, path)
+		return nil, locked, err
+	}
+	return driver, false, nil
+}
+
+// NewOvalDB returns oval db client
+func NewOvalDB(cnf DBClientConf) (driver ovaldb.DB, locked bool, err error) {
+	if config.Conf.OvalDict.IsFetchViaHTTP() {
+		return nil, false, nil
+	}
+	path := cnf.OvalDictCnf.URL
+	if cnf.OvalDictCnf.Type == "sqlite3" {
+		path = cnf.OvalDictCnf.SQLite3Path
+
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			util.Log.Warnf("--ovaldb-path=%s is not found. It's recommended to use OVAL to improve scanning accuracy. For details, see https://github.com/kotakanbe/goval-dictionary#usage", path)
+			return nil, false, nil
+		}
+	}
+
+	util.Log.Debugf("Open oval-dictionary db (%s): %s", cnf.OvalDictCnf.Type, path)
+	driver, locked, err = ovaldb.NewDB("", cnf.OvalDictCnf.Type, path, cnf.DebugSQL)
+	if err != nil {
+		err = fmt.Errorf("Failed to new OVAL DB. err: %s", err)
+		if locked {
+			return nil, true, err
+		}
+		return nil, false, err
+	}
+	return driver, false, nil
+}
+
+// NewGostDB returns db client for Gost
+func NewGostDB(cnf DBClientConf) (driver gostdb.DB, locked bool, err error) {
+	if config.Conf.Gost.IsFetchViaHTTP() {
+		return nil, false, nil
+	}
+	path := cnf.GostCnf.URL
+	if cnf.GostCnf.Type == "sqlite3" {
+		path = cnf.GostCnf.SQLite3Path
+
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			util.Log.Warnf("--gostdb-path=%s is not found. If the scan target server is Debian, RHEL or CentOS, it's recommended to use gost to improve scanning accuracy. To use gost database, see https://github.com/knqyf263/gost#fetch-redhat", path)
+			return nil, false, nil
+		}
+	}
+
+	util.Log.Debugf("Open gost db (%s): %s", cnf.GostCnf.Type, path)
+	if driver, locked, err = gostdb.NewDB(cnf.GostCnf.Type, path, cnf.DebugSQL); err != nil {
+		if locked {
+			util.Log.Errorf("gostDB is locked: %s", err)
+			return nil, true, err
+		}
+		return nil, false, err
+	}
+	return driver, false, nil
+}
+
+// NewExploitDB returns db client for Exploit
+func NewExploitDB(cnf DBClientConf) (driver exploitdb.DB, locked bool, err error) {
+	if config.Conf.Exploit.IsFetchViaHTTP() {
+		return nil, false, nil
+	}
+	path := cnf.ExploitCnf.URL
+	if cnf.ExploitCnf.Type == "sqlite3" {
+		path = cnf.ExploitCnf.SQLite3Path
+
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			util.Log.Warnf("--exploitdb-path=%s is not found. It's recommended to use exploit to improve scanning accuracy. To use exploit db database, see https://github.com/mozqnet/go-exploitdb", path)
+			return nil, false, nil
+		}
+	}
+
+	util.Log.Debugf("Open exploit db (%s): %s", cnf.ExploitCnf.Type, path)
+	if driver, locked, err = exploitdb.NewDB(cnf.ExploitCnf.Type, path, cnf.DebugSQL); err != nil {
+		if locked {
+			util.Log.Errorf("exploitDB is locked: %s", err)
+			return nil, true, err
+		}
+		return nil, false, err
+	}
+	return driver, false, nil
+}
+
+// CloseDB close dbs
+func (d DBClient) CloseDB() {
+	if d.CveDB != nil {
+		if err := d.CveDB.CloseDB(); err != nil {
+			util.Log.Errorf("Failed to close DB: %s", err)
+		}
+	}
+	if d.OvalDB != nil {
+		if err := d.OvalDB.CloseDB(); err != nil {
+			util.Log.Errorf("Failed to close DB: %s", err)
+		}
+	}
+}

report/email.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

report/email_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

report/hipchat.go

@@ -0,0 +1,74 @@
+package report
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+)
+
+// HipChatWriter send report to HipChat
+type HipChatWriter struct{}
+
+func (w HipChatWriter) Write(rs ...models.ScanResult) (err error) {
+	conf := config.Conf.HipChat
+
+	for _, r := range rs {
+		serverInfo := fmt.Sprintf("%s", r.ServerInfo())
+		if err = postMessage(conf.Room, conf.AuthToken, serverInfo); err != nil {
+			return err
+		}
+
+		for _, vinfo := range r.ScannedCves {
+			maxCvss := vinfo.MaxCvssScore()
+			severity := strings.ToUpper(maxCvss.Value.Severity)
+			if severity == "" {
+				severity = "?"
+			}
+
+			message := fmt.Sprintf(`<a href="https://nvd.nist.gov/vuln/detail\%s"> %s </a> <br/>%s (%s)<br/>%s`,
+				vinfo.CveID,
+				vinfo.CveID,
+				strconv.FormatFloat(maxCvss.Value.Score, 'f', 1, 64),
+				severity,
+				vinfo.Summaries(config.Conf.Lang, r.Family)[0].Value,
+			)
+
+			if err = postMessage(conf.Room, conf.AuthToken, message); err != nil {
+				return err
+			}
+		}
+
+	}
+	return nil
+}
+
+func postMessage(room, token, message string) error {
+	uri := fmt.Sprintf("https://api.hipchat.com/v2/room/%s/notification?auth_token=%s", room, token)
+
+	payload := url.Values{
+		"color":          {"purple"},
+		"message_format": {"html"},
+		"message":        {message},
+	}
+	reqs, err := http.NewRequest("POST", uri, strings.NewReader(payload.Encode()))
+	if err != nil {
+		return err
+	}
+
+	reqs.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+	client := &http.Client{}
+
+	resp, err := client.Do(reqs)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	return nil
+}

report/hipchat_test.go

@@ -0,0 +1 @@
+package report

report/http.go

@@ -0,0 +1,62 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package report
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+
+	"github.com/pkg/errors"
+
+	c "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+)
+
+// HTTPRequestWriter writes results to HTTP request
+type HTTPRequestWriter struct{}
+
+// Write sends results as HTTP response
+func (w HTTPRequestWriter) Write(rs ...models.ScanResult) (err error) {
+	for _, r := range rs {
+		b := new(bytes.Buffer)
+		json.NewEncoder(b).Encode(r)
+		_, err = http.Post(c.Conf.HTTP.URL, "application/json; charset=utf-8", b)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// HTTPResponseWriter writes results to HTTP response
+type HTTPResponseWriter struct {
+	Writer http.ResponseWriter
+}
+
+// Write sends results as HTTP response
+func (w HTTPResponseWriter) Write(rs ...models.ScanResult) (err error) {
+	res, err := json.Marshal(rs)
+	if err != nil {
+		return errors.Wrap(err, "Failed to marshal scah results")
+	}
+	w.Writer.Header().Set("Content-Type", "application/json")
+	_, err = w.Writer.Write(res)
+
+	return err
+}

report/localfile.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -72,7 +72,7 @@ func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) {
 			}
 		}
 
-		if c.Conf.FormatShortText {
+		if c.Conf.FormatList {
 			var p string
 			if c.Conf.Diff {
 				p = path + "_short_diff.txt"
@@ -81,7 +81,7 @@ func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) {
 			}
 
 			if err := writeFile(
-				p, []byte(formatShortPlainText(r)), 0600); err != nil {
+				p, []byte(formatList(r)), 0600); err != nil {
 				return fmt.Errorf(
 					"Failed to write text files. path: %s, err: %s", p, err)
 			}
@@ -129,13 +129,7 @@ func writeFile(path string, data []byte, perm os.FileMode) error {
 		if data, err = gz(data); err != nil {
 			return err
 		}
-		path = path + ".gz"
+		path += ".gz"
 	}
-
-	if err := ioutil.WriteFile(
-		path, []byte(data), perm); err != nil {
-		return err
-	}
-
-	return nil
+	return ioutil.WriteFile(path, []byte(data), perm)
 }

report/report.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,13 +18,30 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package report
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
 	"time"
 
+	"github.com/BurntSushi/toml"
 	c "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/contrib/owasp-dependency-check/parser"
+	"github.com/future-architect/vuls/cwe"
+	"github.com/future-architect/vuls/exploit"
+	"github.com/future-architect/vuls/gost"
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/oval"
 	"github.com/future-architect/vuls/util"
+	"github.com/hashicorp/uuid"
+	gostdb "github.com/knqyf263/gost/db"
+	cvedb "github.com/kotakanbe/go-cve-dictionary/db"
+	ovaldb "github.com/kotakanbe/goval-dictionary/db"
+	exploitdb "github.com/mozqnet/go-exploitdb/db"
 )
 
 const (
@@ -33,16 +50,50 @@ const (
 )
 
 // FillCveInfos fills CVE Detailed Information
-func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
-	var filled []models.ScanResult
+func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
+	var filledResults []models.ScanResult
 	reportedAt := time.Now()
+	hostname, _ := os.Hostname()
 	for _, r := range rs {
 		if c.Conf.RefreshCve || needToRefreshCve(r) {
-			if err := FillCveInfo(&r); err != nil {
+			r.ScannedCves = models.VulnInfos{}
+			cpeURIs := []string{}
+			if len(r.Container.ContainerID) == 0 {
+				cpeURIs = c.Conf.Servers[r.ServerName].CpeNames
+				owaspDCXMLPath := c.Conf.Servers[r.ServerName].OwaspDCXMLPath
+				if owaspDCXMLPath != "" {
+					cpes, err := parser.Parse(owaspDCXMLPath)
+					if err != nil {
+						return nil, fmt.Errorf("Failed to read OWASP Dependency Check XML: %s, %s, %s",
+							r.ServerName, owaspDCXMLPath, err)
+					}
+					cpeURIs = append(cpeURIs, cpes...)
+				}
+			} else {
+				if s, ok := c.Conf.Servers[r.ServerName]; ok {
+					if con, ok := s.Containers[r.Container.Name]; ok {
+						cpeURIs = con.Cpes
+						owaspDCXMLPath := con.OwaspDCXMLPath
+						if owaspDCXMLPath != "" {
+							cpes, err := parser.Parse(owaspDCXMLPath)
+							if err != nil {
+								return nil, fmt.Errorf("Failed to read OWASP Dependency Check XML: %s, %s, %s",
+									r.ServerInfo(), owaspDCXMLPath, err)
+							}
+							cpeURIs = append(cpeURIs, cpes...)
+						}
+					}
+				}
+			}
+
+			if err := FillCveInfo(dbclient, &r, cpeURIs); err != nil {
 				return nil, err
 			}
 			r.Lang = c.Conf.Lang
 			r.ReportedAt = reportedAt
+			r.ReportedVersion = c.Version
+			r.ReportedRevision = c.Revision
+			r.ReportedBy = hostname
 			r.Config.Report = c.Conf
 			r.Config.Report.Servers = map[string]c.ServerInfo{
 				r.ServerName: c.Conf.Servers[r.ServerName],
@@ -50,37 +101,38 @@ func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, erro
 			if err := overwriteJSONFile(dir, r); err != nil {
 				return nil, fmt.Errorf("Failed to write JSON: %s", err)
 			}
-			filled = append(filled, r)
+			filledResults = append(filledResults, r)
 		} else {
 			util.Log.Debugf("No need to refresh")
-			filled = append(filled, r)
+			filledResults = append(filledResults, r)
 		}
 	}
 
 	if c.Conf.Diff {
-		previous, err := loadPrevious(filled)
+		prevs, err := loadPrevious(filledResults)
 		if err != nil {
 			return nil, err
 		}
 
-		diff, err := diff(filled, previous)
+		diff, err := diff(filledResults, prevs)
 		if err != nil {
 			return nil, err
 		}
-		filled = []models.ScanResult{}
+		filledResults = []models.ScanResult{}
 		for _, r := range diff {
-			if err := fillCveDetail(&r); err != nil {
+			if err := fillCveDetail(dbclient.CveDB, &r); err != nil {
 				return nil, err
 			}
-			filled = append(filled, r)
+			filledResults = append(filledResults, r)
 		}
 	}
 
 	filtered := []models.ScanResult{}
-	for _, r := range filled {
+	for _, r := range filledResults {
 		r = r.FilterByCvssOver(c.Conf.CvssScoreOver)
-		r = r.FilterIgnoreCves(c.Conf.Servers[r.ServerName].IgnoreCves)
+		r = r.FilterIgnoreCves()
 		r = r.FilterUnfixed()
+		r = r.FilterIgnorePkgs()
 		if c.Conf.IgnoreUnscoredCves {
 			r.ScannedCves = r.ScannedCves.FindScoredVulns()
 		}
@@ -90,48 +142,81 @@ func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, erro
 }
 
 // FillCveInfo fill scanResult with cve info.
-func FillCveInfo(r *models.ScanResult) error {
+func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string) error {
 	util.Log.Debugf("need to refresh")
 
-	util.Log.Infof("Fill CVE detailed information with OVAL")
-	if err := FillWithOval(r); err != nil {
-		return fmt.Errorf("Failed to fill OVAL information: %s", err)
+	nCVEs, err := FillWithOval(dbclient.OvalDB, r)
+	if err != nil {
+		return fmt.Errorf("Failed to fill with OVAL: %s", err)
 	}
+	util.Log.Infof("%s: %d CVEs are detected with OVAL",
+		r.FormatServerName(), nCVEs)
+
+	for i, v := range r.ScannedCves {
+		for j, p := range v.AffectedPackages {
+			if p.NotFixedYet && p.FixState == "" {
+				p.FixState = "Not fixed yet"
+				r.ScannedCves[i].AffectedPackages[j] = p
+			}
+		}
+	}
+
+	nCVEs, err = fillVulnByCpeURIs(dbclient.CveDB, r, cpeURIs)
+	if err != nil {
+		return fmt.Errorf("Failed to detect vulns of %s: %s", cpeURIs, err)
+	}
+	util.Log.Infof("%s: %d CVEs are detected with CPE", r.FormatServerName(), nCVEs)
+
+	nCVEs, err = FillWithGost(dbclient.GostDB, r)
+	if err != nil {
+		return fmt.Errorf("Failed to fill with gost: %s", err)
+	}
+	util.Log.Infof("%s: %d unfixed CVEs are detected with gost",
+		r.FormatServerName(), nCVEs)
 
 	util.Log.Infof("Fill CVE detailed information with CVE-DB")
-	if err := fillWithCveDB(r); err != nil {
-		return fmt.Errorf("Failed to fill CVE information: %s", err)
+	if err := fillCveDetail(dbclient.CveDB, r); err != nil {
+		return fmt.Errorf("Failed to fill with CVE: %s", err)
 	}
 
-	for cveID := range r.ScannedCves {
-		vinfo := r.ScannedCves[cveID]
-		r.ScannedCves[cveID] = *vinfo.NilToEmpty()
+	util.Log.Infof("Fill exploit information with Exploit-DB")
+	nExploitCve, err := FillWithExploit(dbclient.ExploitDB, r)
+	if err != nil {
+		return fmt.Errorf("Failed to fill with exploit: %s", err)
 	}
+	util.Log.Infof("%s: %d exploits are detected",
+		r.FormatServerName(), nExploitCve)
+
+	fillCweDict(r)
 	return nil
 }
 
-// fillCveDetail fetches NVD, JVN from CVE Database, and then set to fields.
-func fillCveDetail(r *models.ScanResult) error {
+// fillCveDetail fetches NVD, JVN from CVE Database
+func fillCveDetail(driver cvedb.DB, r *models.ScanResult) error {
 	var cveIDs []string
 	for _, v := range r.ScannedCves {
 		cveIDs = append(cveIDs, v.CveID)
 	}
 
-	ds, err := CveClient.FetchCveDetails(cveIDs)
+	ds, err := CveClient.FetchCveDetails(driver, cveIDs)
 	if err != nil {
 		return err
 	}
 	for _, d := range ds {
-		nvd := models.ConvertNvdToModel(d.CveID, d.Nvd)
+		nvd := models.ConvertNvdJSONToModel(d.CveID, d.NvdJSON)
+		if nvd == nil {
+			nvd = models.ConvertNvdXMLToModel(d.CveID, d.NvdXML)
+		}
 		jvn := models.ConvertJvnToModel(d.CveID, d.Jvn)
+
 		for cveID, vinfo := range r.ScannedCves {
 			if vinfo.CveID == d.CveID {
 				if vinfo.CveContents == nil {
 					vinfo.CveContents = models.CveContents{}
 				}
-				for _, con := range []models.CveContent{*nvd, *jvn} {
-					if !con.Empty() {
-						vinfo.CveContents[con.Type] = con
+				for _, con := range []*models.CveContent{nvd, jvn} {
+					if con != nil && !con.Empty() {
+						vinfo.CveContents[con.Type] = *con
 					}
 				}
 				r.ScannedCves[cveID] = vinfo
@@ -142,23 +227,11 @@ func fillCveDetail(r *models.ScanResult) error {
 	return nil
 }
 
-func fillWithCveDB(r *models.ScanResult) error {
-	sInfo := c.Conf.Servers[r.ServerName]
-	if err := fillVulnByCpeNames(sInfo.CpeNames, r.ScannedCves); err != nil {
-		return err
-	}
-	if err := fillCveDetail(r); err != nil {
-		return err
-	}
-	return nil
-}
-
-// FillWithOval fetches OVAL database, and then set to fields.
-func FillWithOval(r *models.ScanResult) (err error) {
+// FillWithOval fetches OVAL database
+func FillWithOval(driver ovaldb.DB, r *models.ScanResult) (nCVEs int, err error) {
 	var ovalClient oval.Client
 	var ovalFamily string
 
-	// TODO
 	switch r.Family {
 	case c.Debian:
 		ovalClient = oval.NewDebian()
@@ -184,57 +257,378 @@ func FillWithOval(r *models.ScanResult) (err error) {
 		ovalClient = oval.NewAlpine()
 		ovalFamily = c.Alpine
 	case c.Amazon, c.Raspbian, c.FreeBSD, c.Windows:
-		return nil
+		return 0, nil
 	case c.ServerTypePseudo:
-		return nil
+		return 0, nil
 	default:
-		return fmt.Errorf("OVAL for %s is not implemented yet", r.Family)
+		if r.Family == "" {
+			return 0, fmt.Errorf("Probably an error occurred during scanning. Check the error message")
+		}
+		return 0, fmt.Errorf("OVAL for %s is not implemented yet", r.Family)
 	}
 
-	util.Log.Debugf("Check whether oval is already fetched: %s %s",
-		ovalFamily, r.Release)
-	ok, err := ovalClient.CheckIfOvalFetched(ovalFamily, r.Release)
+	if !c.Conf.OvalDict.IsFetchViaHTTP() {
+		if driver == nil {
+			return 0, nil
+		}
+		if err = driver.NewOvalDB(ovalFamily); err != nil {
+			return 0, fmt.Errorf("Failed to New Oval DB. err: %s", err)
+		}
+	}
+
+	util.Log.Debugf("Check whether oval fetched: %s %s", ovalFamily, r.Release)
+	ok, err := ovalClient.CheckIfOvalFetched(driver, ovalFamily, r.Release)
 	if err != nil {
-		return err
+		return 0, err
 	}
 	if !ok {
 		util.Log.Warnf("OVAL entries of %s %s are not found. It's recommended to use OVAL to improve scanning accuracy. For details, see https://github.com/kotakanbe/goval-dictionary#usage , Then report with --ovaldb-path or --ovaldb-url flag", ovalFamily, r.Release)
-		return nil
+		return 0, nil
 	}
 
-	_, err = ovalClient.CheckIfOvalFresh(ovalFamily, r.Release)
+	_, err = ovalClient.CheckIfOvalFresh(driver, ovalFamily, r.Release)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
-	if err := ovalClient.FillWithOval(r); err != nil {
-		return err
-	}
-	return nil
+	return ovalClient.FillWithOval(driver, r)
 }
 
-func fillVulnByCpeNames(cpeNames []string, scannedVulns models.VulnInfos) error {
-	for _, name := range cpeNames {
-		details, err := CveClient.FetchCveDetailsByCpeName(name)
+// FillWithGost fills CVEs with gost dataabase
+// https://github.com/knqyf263/gost
+func FillWithGost(driver gostdb.DB, r *models.ScanResult) (nCVEs int, err error) {
+	gostClient := gost.NewClient(r.Family)
+	// TODO chekc if fetched
+	// TODO chekc if fresh enough
+	return gostClient.FillWithGost(driver, r)
+}
+
+// FillWithExploit fills Exploits with exploit dataabase
+// https://github.com/mozqnet/go-exploitdb
+func FillWithExploit(driver exploitdb.DB, r *models.ScanResult) (nExploitCve int, err error) {
+	// TODO chekc if fetched
+	// TODO chekc if fresh enough
+	return exploit.FillWithExploit(driver, r)
+}
+
+func fillVulnByCpeURIs(driver cvedb.DB, r *models.ScanResult, cpeURIs []string) (nCVEs int, err error) {
+	for _, name := range cpeURIs {
+		details, err := CveClient.FetchCveDetailsByCpeName(driver, name)
 		if err != nil {
-			return err
+			return 0, err
 		}
 		for _, detail := range details {
-			if val, ok := scannedVulns[detail.CveID]; ok {
-				names := val.CpeNames
+			if val, ok := r.ScannedCves[detail.CveID]; ok {
+				names := val.CpeURIs
 				names = util.AppendIfMissing(names, name)
-				val.CpeNames = names
-				val.Confidence = models.CpeNameMatch
-				scannedVulns[detail.CveID] = val
+				val.CpeURIs = names
+				val.Confidences.AppendIfMissing(models.CpeNameMatch)
+				r.ScannedCves[detail.CveID] = val
 			} else {
 				v := models.VulnInfo{
-					CveID:      detail.CveID,
-					CpeNames:   []string{name},
-					Confidence: models.CpeNameMatch,
+					CveID:       detail.CveID,
+					CpeURIs:     []string{name},
+					Confidences: models.Confidences{models.CpeNameMatch},
 				}
-				scannedVulns[detail.CveID] = v
+				r.ScannedCves[detail.CveID] = v
+				nCVEs++
 			}
 		}
 	}
-	return nil
+	return nCVEs, nil
+}
+
+func fillCweDict(r *models.ScanResult) {
+	uniqCweIDMap := map[string]bool{}
+	for _, vinfo := range r.ScannedCves {
+		for _, cont := range vinfo.CveContents {
+			for _, id := range cont.CweIDs {
+				if strings.HasPrefix(id, "CWE-") {
+					id = strings.TrimPrefix(id, "CWE-")
+					uniqCweIDMap[id] = true
+				}
+			}
+		}
+	}
+
+	// TODO check the format of CWEID, clean CWEID
+	// JVN, NVD XML, JSON, OVALs
+
+	dict := map[string]models.CweDictEntry{}
+	for id := range uniqCweIDMap {
+		entry := models.CweDictEntry{}
+		if e, ok := cwe.CweDictEn[id]; ok {
+			if rank, ok := cwe.OwaspTopTen2017[id]; ok {
+				entry.OwaspTopTen2017 = rank
+			}
+			entry.En = &e
+		} else {
+			util.Log.Debugf("CWE-ID %s is not found in English CWE Dict", id)
+			entry.En = &cwe.Cwe{CweID: id}
+		}
+
+		if c.Conf.Lang == "ja" {
+			if e, ok := cwe.CweDictJa[id]; ok {
+				if rank, ok := cwe.OwaspTopTen2017[id]; ok {
+					entry.OwaspTopTen2017 = rank
+				}
+				entry.Ja = &e
+			} else {
+				util.Log.Debugf("CWE-ID %s is not found in Japanese CWE Dict", id)
+				entry.Ja = &cwe.Cwe{CweID: id}
+			}
+		}
+		dict[id] = entry
+	}
+	r.CweDict = dict
+	return
+}
+
+const reUUID = "[\\da-f]{8}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{12}"
+
+// EnsureUUIDs generate a new UUID of the scan target server if UUID is not assigned yet.
+// And then set the generated UUID to config.toml and scan results.
+func EnsureUUIDs(configPath string, results models.ScanResults) error {
+	// Sort Host->Container
+	sort.Slice(results, func(i, j int) bool {
+		if results[i].ServerName == results[j].ServerName {
+			return results[i].Container.ContainerID < results[j].Container.ContainerID
+		}
+		return results[i].ServerName < results[j].ServerName
+	})
+
+	for i, r := range results {
+		server := c.Conf.Servers[r.ServerName]
+		if server.UUIDs == nil {
+			server.UUIDs = map[string]string{}
+		}
+
+		name := ""
+		if r.IsContainer() {
+			name = fmt.Sprintf("%s@%s", r.Container.Name, r.ServerName)
+
+			// Scanning with the -containers-only flag at scan time, the UUID of Container Host may not be generated,
+			// so check it. Otherwise create a UUID of the Container Host and set it.
+			serverUUID := ""
+			if id, ok := server.UUIDs[r.ServerName]; !ok {
+				serverUUID = uuid.GenerateUUID()
+			} else {
+				matched, err := regexp.MatchString(reUUID, id)
+				if !matched || err != nil {
+					serverUUID = uuid.GenerateUUID()
+				}
+			}
+			if serverUUID != "" {
+				server.UUIDs[r.ServerName] = serverUUID
+			}
+		} else {
+			name = r.ServerName
+		}
+
+		if id, ok := server.UUIDs[name]; ok {
+			matched, err := regexp.MatchString(reUUID, id)
+			if !matched || err != nil {
+				util.Log.Warnf("UUID is invalid. Re-generate UUID %s: %s", id, err)
+			} else {
+				if r.IsContainer() {
+					results[i].Container.UUID = id
+					results[i].ServerUUID = server.UUIDs[r.ServerName]
+				} else {
+					results[i].ServerUUID = id
+				}
+				// continue if the UUID has already assigned and valid
+				continue
+			}
+		}
+
+		// Generate a new UUID and set to config and scan result
+		id := uuid.GenerateUUID()
+		server.UUIDs[name] = id
+		server = cleanForTOMLEncoding(server, c.Conf.Default)
+		c.Conf.Servers[r.ServerName] = server
+
+		if r.IsContainer() {
+			results[i].Container.UUID = id
+			results[i].ServerUUID = server.UUIDs[r.ServerName]
+		} else {
+			results[i].ServerUUID = id
+		}
+	}
+
+	for name, server := range c.Conf.Servers {
+		server = cleanForTOMLEncoding(server, c.Conf.Default)
+		c.Conf.Servers[name] = server
+	}
+
+	email := &c.Conf.EMail
+	if email.SMTPAddr == "" {
+		email = nil
+	}
+
+	slack := &c.Conf.Slack
+	if slack.HookURL == "" {
+		slack = nil
+	}
+
+	cveDict := &c.Conf.CveDict
+	ovalDict := &c.Conf.OvalDict
+	gost := &c.Conf.Gost
+	exploit := &c.Conf.Exploit
+	http := &c.Conf.HTTP
+	if http.URL == "" {
+		http = nil
+	}
+
+	syslog := &c.Conf.Syslog
+	if syslog.Host == "" {
+		syslog = nil
+	}
+
+	aws := &c.Conf.AWS
+	if aws.S3Bucket == "" {
+		aws = nil
+	}
+
+	azure := &c.Conf.Azure
+	if azure.AccountName == "" {
+		azure = nil
+	}
+
+	stride := &c.Conf.Stride
+	if stride.HookURL == "" {
+		stride = nil
+	}
+
+	hipChat := &c.Conf.HipChat
+	if hipChat.AuthToken == "" {
+		hipChat = nil
+	}
+
+	chatWork := &c.Conf.ChatWork
+	if chatWork.APIToken == "" {
+		chatWork = nil
+	}
+
+	saas := &c.Conf.Saas
+	if saas.GroupID == 0 {
+		saas = nil
+	}
+
+	c := struct {
+		CveDict  *c.GoCveDictConf `toml:"cveDict"`
+		OvalDict *c.GovalDictConf `toml:"ovalDict"`
+		Gost     *c.GostConf      `toml:"gost"`
+		Exploit  *c.ExploitConf   `toml:"exploit"`
+		Slack    *c.SlackConf     `toml:"slack"`
+		Email    *c.SMTPConf      `toml:"email"`
+		HTTP     *c.HTTPConf      `toml:"http"`
+		Syslog   *c.SyslogConf    `toml:"syslog"`
+		AWS      *c.AWS           `toml:"aws"`
+		Azure    *c.Azure         `toml:"azure"`
+		Stride   *c.StrideConf    `toml:"stride"`
+		HipChat  *c.HipChatConf   `toml:"hipChat"`
+		ChatWork *c.ChatWorkConf  `toml:"chatWork"`
+		Saas     *c.SaasConf      `toml:"saas"`
+
+		Default c.ServerInfo            `toml:"default"`
+		Servers map[string]c.ServerInfo `toml:"servers"`
+	}{
+		CveDict:  cveDict,
+		OvalDict: ovalDict,
+		Gost:     gost,
+		Exploit:  exploit,
+		Slack:    slack,
+		Email:    email,
+		HTTP:     http,
+		Syslog:   syslog,
+		AWS:      aws,
+		Azure:    azure,
+		Stride:   stride,
+		HipChat:  hipChat,
+		ChatWork: chatWork,
+		Saas:     saas,
+
+		Default: c.Conf.Default,
+		Servers: c.Conf.Servers,
+	}
+
+	// rename the current config.toml to config.toml.bak
+	info, err := os.Lstat(configPath)
+	if err != nil {
+		return fmt.Errorf("Failed to lstat %s: %s", configPath, err)
+	}
+	realPath := configPath
+	if info.Mode()&os.ModeSymlink == os.ModeSymlink {
+		if realPath, err = os.Readlink(configPath); err != nil {
+			return fmt.Errorf("Failed to Read link %s: %s", configPath, err)
+		}
+	}
+	if err := os.Rename(realPath, realPath+".bak"); err != nil {
+		return fmt.Errorf("Failed to rename %s: %s", configPath, err)
+	}
+
+	var buf bytes.Buffer
+	if err := toml.NewEncoder(&buf).Encode(c); err != nil {
+		return fmt.Errorf("Failed to encode to toml: %s", err)
+	}
+	str := strings.Replace(buf.String(), "\n  [", "\n\n  [", -1)
+	str = fmt.Sprintf("%s\n\n%s",
+		"# See REAME for details: https://vuls.io/docs/en/usage-settings.html",
+		str)
+
+	return ioutil.WriteFile(realPath, []byte(str), 0600)
+}
+
+func cleanForTOMLEncoding(server c.ServerInfo, def c.ServerInfo) c.ServerInfo {
+	if reflect.DeepEqual(server.Optional, def.Optional) {
+		server.Optional = nil
+	}
+
+	if def.User == server.User {
+		server.User = ""
+	}
+
+	if def.Host == server.Host {
+		server.Host = ""
+	}
+
+	if def.Port == server.Port {
+		server.Port = ""
+	}
+
+	if def.KeyPath == server.KeyPath {
+		server.KeyPath = ""
+	}
+
+	if reflect.DeepEqual(server.ScanMode, def.ScanMode) {
+		server.ScanMode = nil
+	}
+
+	if def.Type == server.Type {
+		server.Type = ""
+	}
+
+	if reflect.DeepEqual(server.CpeNames, def.CpeNames) {
+		server.CpeNames = nil
+	}
+
+	if def.OwaspDCXMLPath == server.OwaspDCXMLPath {
+		server.OwaspDCXMLPath = ""
+	}
+
+	if reflect.DeepEqual(server.IgnoreCves, def.IgnoreCves) {
+		server.IgnoreCves = nil
+	}
+
+	if reflect.DeepEqual(server.Enablerepo, def.Enablerepo) {
+		server.Enablerepo = nil
+	}
+
+	for k, v := range def.Optional {
+		if vv, ok := server.Optional[k]; ok && v == vv {
+			delete(server.Optional, k)
+		}
+	}
+
+	return server
 }

report/s3.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -41,10 +41,10 @@ type S3Writer struct{}
 
 func getS3() *s3.S3 {
 	Config := &aws.Config{
-		Region: aws.String(c.Conf.AwsRegion),
+		Region: aws.String(c.Conf.AWS.Region),
 		Credentials: credentials.NewChainCredentials([]credentials.Provider{
 			&credentials.EnvProvider{},
-			&credentials.SharedCredentialsProvider{Filename: "", Profile: c.Conf.AwsProfile},
+			&credentials.SharedCredentialsProvider{Filename: "", Profile: c.Conf.AWS.Profile},
 			&ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())},
 		}),
 	}
@@ -82,9 +82,9 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) {
 			}
 		}
 
-		if c.Conf.FormatShortText {
+		if c.Conf.FormatList {
 			k := key + "_short.txt"
-			text := formatShortPlainText(r)
+			text := formatList(r)
 			if err := putObject(svc, k, []byte(text)); err != nil {
 				return err
 			}
@@ -120,20 +120,20 @@ func CheckIfBucketExists() error {
 	if err != nil {
 		return fmt.Errorf(
 			"Failed to list buckets. err: %s, profile: %s, region: %s",
-			err, c.Conf.AwsProfile, c.Conf.AwsRegion)
+			err, c.Conf.AWS.Profile, c.Conf.AWS.Region)
 	}
 
 	found := false
 	for _, bucket := range result.Buckets {
-		if *bucket.Name == c.Conf.S3Bucket {
+		if *bucket.Name == c.Conf.AWS.S3Bucket {
 			found = true
 			break
 		}
 	}
 	if !found {
 		return fmt.Errorf(
-			"Failed to find the buckets. profile: %s, region: %s, bukdet: %s",
-			c.Conf.AwsProfile, c.Conf.AwsRegion, c.Conf.S3Bucket)
+			"Failed to find the buckets. profile: %s, region: %s, bucket: %s",
+			c.Conf.AWS.Profile, c.Conf.AWS.Region, c.Conf.AWS.S3Bucket)
 	}
 	return nil
 }
@@ -144,16 +144,22 @@ func putObject(svc *s3.S3, k string, b []byte) error {
 		if b, err = gz(b); err != nil {
 			return err
 		}
-		k = k + ".gz"
+		k += ".gz"
 	}
 
-	if _, err := svc.PutObject(&s3.PutObjectInput{
-		Bucket: aws.String(c.Conf.S3Bucket),
-		Key:    aws.String(path.Join(c.Conf.S3ResultsDir, k)),
+	putObjectInput := &s3.PutObjectInput{
+		Bucket: aws.String(c.Conf.AWS.S3Bucket),
+		Key:    aws.String(path.Join(c.Conf.AWS.S3ResultsDir, k)),
 		Body:   bytes.NewReader(b),
-	}); err != nil {
+	}
+
+	if c.Conf.AWS.S3ServerSideEncryption != "" {
+		putObjectInput.ServerSideEncryption = aws.String(c.Conf.AWS.S3ServerSideEncryption)
+	}
+
+	if _, err := svc.PutObject(putObjectInput); err != nil {
 		return fmt.Errorf("Failed to upload data to %s/%s, %s",
-			c.Conf.S3Bucket, k, err)
+			c.Conf.AWS.S3Bucket, k, err)
 	}
 	return nil
 }

report/saas.go

@@ -0,0 +1,153 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Corporation , Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package report
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/sts"
+	c "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+)
+
+// SaasWriter writes results to SaaS
+type SaasWriter struct{}
+
+// TempCredential : TempCredential
+type TempCredential struct {
+	Credential   *sts.Credentials `json:"Credential"`
+	S3Bucket     string           `json:"S3Bucket"`
+	S3ResultsDir string           `json:"S3ResultsDir"`
+}
+
+type payload struct {
+	GroupID int    `json:"GroupID"`
+	Token   string `json:"Token"`
+}
+
+// UploadSaas : UploadSaas
+func (w SaasWriter) Write(rs ...models.ScanResult) (err error) {
+	// dir string, configPath string, config *c.Config
+	if len(rs) == 0 {
+		return nil
+	}
+
+	payload := payload{
+		GroupID: c.Conf.Saas.GroupID,
+		Token:   c.Conf.Saas.Token,
+	}
+
+	var body []byte
+	if body, err = json.Marshal(payload); err != nil {
+		return fmt.Errorf("Failed to Marshal to JSON: %s", err)
+	}
+
+	var req *http.Request
+	if req, err = http.NewRequest("POST", c.Conf.Saas.URL, bytes.NewBuffer(body)); err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+
+	proxy := c.Conf.HTTPProxy
+	var client http.Client
+	if proxy != "" {
+		proxyURL, _ := url.Parse(proxy)
+		client = http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(proxyURL),
+			},
+		}
+	} else {
+		client = http.Client{}
+	}
+
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("Failed to get Credential. Request JSON : %s,", string(body))
+	}
+
+	var t []byte
+	if t, err = ioutil.ReadAll(resp.Body); err != nil {
+		return err
+	}
+
+	var tempCredential TempCredential
+	if err = json.Unmarshal(t, &tempCredential); err != nil {
+		return fmt.Errorf("Failed to unmarshal saas credential file. err : %s", err)
+	}
+
+	credential := credentials.NewStaticCredentialsFromCreds(credentials.Value{
+		AccessKeyID:     *tempCredential.Credential.AccessKeyId,
+		SecretAccessKey: *tempCredential.Credential.SecretAccessKey,
+		SessionToken:    *tempCredential.Credential.SessionToken,
+	})
+
+	var sess *session.Session
+	if sess, err = session.NewSession(&aws.Config{
+		Credentials: credential,
+		Region:      aws.String("ap-northeast-1"),
+	}); err != nil {
+		return fmt.Errorf("Failed to new aws session. err : %s", err)
+	}
+
+	svc := s3.New(sess)
+	for _, r := range rs {
+		s3Key := renameKeyNameUTC(r.ScannedAt, r.ServerUUID, r.Container)
+		var b []byte
+		if b, err = json.Marshal(r); err != nil {
+			return fmt.Errorf("Failed to Marshal to JSON: %s", err)
+		}
+		util.Log.Infof("Uploading...: ServerName: %s, ", r.ServerName)
+		putObjectInput := &s3.PutObjectInput{
+			Bucket: aws.String(tempCredential.S3Bucket),
+			Key:    aws.String(path.Join(tempCredential.S3ResultsDir, s3Key)),
+			Body:   bytes.NewReader(b),
+		}
+
+		if _, err := svc.PutObject(putObjectInput); err != nil {
+			return fmt.Errorf("Failed to upload data to %s/%s, %s",
+				tempCredential.S3Bucket, s3Key, err)
+		}
+	}
+	return nil
+}
+
+func renameKeyNameUTC(scannedAt time.Time, uuid string, container models.Container) string {
+	timestr := scannedAt.UTC().Format(time.RFC3339)
+	if len(container.ContainerID) == 0 {
+		return fmt.Sprintf("%s/%s.json", timestr, uuid)
+	}
+	return fmt.Sprintf("%s/%s@%s.json", timestr, container.UUID, uuid)
+}

report/slack.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -61,12 +61,8 @@ func (w SlackWriter) Write(rs ...models.ScanResult) (err error) {
 		}
 
 		if 0 < len(r.Errors) {
-			serverInfo := fmt.Sprintf("*%s*", r.ServerInfo())
-			notifyUsers := getNotifyUsers(config.Conf.Slack.NotifyUsers)
-			txt := fmt.Sprintf("%s\n%s\nError: %s",
-				notifyUsers, serverInfo, r.Errors)
 			msg := message{
-				Text:      txt,
+				Text:      msgText(r),
 				Username:  conf.AuthUser,
 				IconEmoji: conf.IconEmoji,
 				Channel:   channel,
@@ -179,10 +175,22 @@ func msgText(r models.ScanResult) string {
 		notifyUsers = getNotifyUsers(config.Conf.Slack.NotifyUsers)
 	}
 	serverInfo := fmt.Sprintf("*%s*", r.ServerInfo())
-	return fmt.Sprintf("%s\n%s\n>%s",
+
+	if 0 < len(r.Errors) {
+		return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\nError: %s",
+			notifyUsers,
+			serverInfo,
+			r.ScannedCves.FormatCveSummary(),
+			r.ScannedCves.FormatFixedStatus(r.Packages),
+			r.FormatUpdatablePacksSummary(),
+			r.Errors)
+	}
+	return fmt.Sprintf("%s\n%s\n%s\n%s\n%s",
 		notifyUsers,
 		serverInfo,
-		r.ScannedCves.FormatCveSummary())
+		r.ScannedCves.FormatCveSummary(),
+		r.ScannedCves.FormatFixedStatus(r.Packages),
+		r.FormatUpdatablePacksSummary())
 }
 
 func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) {
@@ -197,7 +205,7 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) {
 				curent = append(curent, affected.Name)
 			}
 		}
-		for _, n := range vinfo.CpeNames {
+		for _, n := range vinfo.CpeURIs {
 			curent = append(curent, n)
 		}
 
@@ -213,14 +221,14 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) {
 				new = append(new, "?")
 			}
 		}
-		for range vinfo.CpeNames {
+		for range vinfo.CpeURIs {
 			new = append(new, "?")
 		}
 
 		a := slack.Attachment{
 			Title:      vinfo.CveID,
 			TitleLink:  "https://nvd.nist.gov/vuln/detail/" + vinfo.CveID,
-			Text:       attachmentText(vinfo, r.Family),
+			Text:       attachmentText(vinfo, r.Family, r.CweDict, r.Packages),
 			MarkdownIn: []string{"text", "pretext"},
 			Fields: []slack.AttachmentField{
 				{
@@ -235,7 +243,7 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) {
 					Short: true,
 				},
 			},
-			Color: color(vinfo.MaxCvssScore().Value.Score),
+			Color: cvssColor(vinfo.MaxCvssScore().Value.Score),
 		}
 		attaches = append(attaches, a)
 	}
@@ -243,7 +251,7 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) {
 }
 
 // https://api.slack.com/docs/attachments
-func color(cvssScore float64) string {
+func cvssColor(cvssScore float64) string {
 	switch {
 	case 7 <= cvssScore:
 		return "danger"
@@ -256,10 +264,15 @@ func color(cvssScore float64) string {
 	}
 }
 
-func attachmentText(vinfo models.VulnInfo, osFamily string) string {
+func attachmentText(vinfo models.VulnInfo, osFamily string, cweDict map[string]models.CweDictEntry, packs models.Packages) string {
 	maxCvss := vinfo.MaxCvssScore()
 	vectors := []string{}
-	for _, cvss := range vinfo.Cvss2Scores() {
+
+	scores := append(vinfo.Cvss3Scores(), vinfo.Cvss2Scores(osFamily)...)
+	for _, cvss := range scores {
+		if cvss.Value.Severity == "" {
+			continue
+		}
 		calcURL := ""
 		switch cvss.Value.Type {
 		case models.CVSS2:
@@ -273,9 +286,10 @@ func attachmentText(vinfo models.VulnInfo, osFamily string) string {
 		}
 
 		if cont, ok := vinfo.CveContents[cvss.Type]; ok {
-			v := fmt.Sprintf("<%s|%s> (<%s|%s>)",
+			v := fmt.Sprintf("<%s|%s> %s (<%s|%s>)",
 				calcURL,
-				cvss.Value.Format(),
+				fmt.Sprintf("%3.1f/%s", cvss.Value.Score, cvss.Value.Vector),
+				cvss.Value.Severity,
 				cont.SourceLink,
 				cvss.Type)
 			vectors = append(vectors, v)
@@ -288,9 +302,10 @@ func attachmentText(vinfo models.VulnInfo, osFamily string) string {
 						v, k))
 				}
 
-				v := fmt.Sprintf("<%s|%s> (%s)",
+				v := fmt.Sprintf("<%s|%s> %s (%s)",
 					calcURL,
-					cvss.Value.Format(),
+					fmt.Sprintf("%3.1f/%s", cvss.Value.Score, cvss.Value.Vector),
+					cvss.Value.Severity,
 					strings.Join(links, ", "))
 				vectors = append(vectors, v)
 			}
@@ -302,27 +317,42 @@ func attachmentText(vinfo models.VulnInfo, osFamily string) string {
 		severity = "?"
 	}
 
-	return fmt.Sprintf("*%4.1f (%s)* %s\n%s\n```%s```",
+	nwvec := vinfo.AttackVector()
+	if nwvec == "Network" || nwvec == "remote" {
+		nwvec = fmt.Sprintf("*%s*", nwvec)
+	}
+
+	mitigation := ""
+	if vinfo.Mitigations(osFamily)[0].Type != models.Unknown {
+		mitigation = fmt.Sprintf("\nMitigation:\n```%s```\n",
+			vinfo.Mitigations(osFamily)[0].Value)
+	}
+
+	return fmt.Sprintf("*%4.1f (%s)* %s %s\n%s\n```\n%s\n```%s\n%s\n",
 		maxCvss.Value.Score,
 		severity,
-		cweIDs(vinfo, osFamily),
+		nwvec,
+		vinfo.PatchStatus(packs),
 		strings.Join(vectors, "\n"),
 		vinfo.Summaries(config.Conf.Lang, osFamily)[0].Value,
+		mitigation,
+		cweIDs(vinfo, osFamily, cweDict),
 	)
 }
 
-func cweIDs(vinfo models.VulnInfo, osFamily string) string {
+func cweIDs(vinfo models.VulnInfo, osFamily string, cweDict models.CweDict) string {
 	links := []string{}
-	for _, cwe := range vinfo.CveContents.CweIDs(osFamily) {
-		if config.Conf.Lang == "ja" {
-			links = append(links, fmt.Sprintf("<%s|%s>",
-				cweJvnURL(cwe.Value), cwe.Value))
-		} else {
-			links = append(links, fmt.Sprintf("<%s|%s>",
-				cweURL(cwe.Value), cwe.Value))
+	for _, c := range vinfo.CveContents.UniqCweIDs(osFamily) {
+		name, url, top10Rank, top10URL := cweDict.Get(c.Value, osFamily)
+		line := ""
+		if top10Rank != "" {
+			line = fmt.Sprintf("<%s|[OWASP Top %s]>",
+				top10URL, top10Rank)
 		}
+		links = append(links, fmt.Sprintf("%s <%s|%s>: %s",
+			line, url, c.Value, name))
 	}
-	return strings.Join(links, " / ")
+	return strings.Join(links, "\n")
 }
 
 // See testcase

report/stdout.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -44,9 +44,9 @@ func (w StdoutWriter) Write(rs ...models.ScanResult) error {
 		fmt.Print("\n")
 	}
 
-	if c.Conf.FormatShortText {
+	if c.Conf.FormatList {
 		for _, r := range rs {
-			fmt.Println(formatShortPlainText(r))
+			fmt.Println(formatList(r))
 		}
 	}
 

report/stride.go

@@ -0,0 +1,81 @@
+package report
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+
+	"strconv"
+	"strings"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+)
+
+// StrideWriter send report to Stride
+type StrideWriter struct{}
+type strideSender struct{}
+
+func (w StrideWriter) Write(rs ...models.ScanResult) (err error) {
+	conf := config.Conf.Stride
+
+	for _, r := range rs {
+		w := strideSender{}
+
+		serverInfo := fmt.Sprintf("%s", r.ServerInfo())
+		message := fmt.Sprintf(`{"body":{"version":1,"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":" %s  "}]}]}}`,
+			serverInfo,
+		)
+		if err = w.sendMessage(conf.HookURL, conf.AuthToken, message); err != nil {
+			return err
+		}
+
+		for _, vinfo := range r.ScannedCves {
+			maxCvss := vinfo.MaxCvssScore()
+			severity := strings.ToUpper(maxCvss.Value.Severity)
+			if severity == "" {
+				severity = "?"
+			}
+
+			message = fmt.Sprintf(`{"body":{"version":1,"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":" %s ","marks": [ { "type": "link", "attrs": { "href": "https://nvd.nist.gov/vuln/detail/%s", "title": "cve" } } ]}]}]}}`,
+				vinfo.CveID,
+				vinfo.CveID,
+			)
+			if err = w.sendMessage(conf.HookURL, conf.AuthToken, message); err != nil {
+				return err
+			}
+
+			message = fmt.Sprintf(`{"body":{"version":1,"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":" %s (%s) "}]}]}}`,
+				strconv.FormatFloat(maxCvss.Value.Score, 'f', 1, 64),
+				severity,
+			)
+			if err = w.sendMessage(conf.HookURL, conf.AuthToken, message); err != nil {
+				return err
+			}
+
+			message = fmt.Sprintf(`{"body":{"version":1,"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":" %s "}]}]}}`,
+				vinfo.Summaries(config.Conf.Lang, r.Family)[0].Value,
+			)
+			if err = w.sendMessage(conf.HookURL, conf.AuthToken, message); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (w strideSender) sendMessage(uri, token, jsonStr string) error {
+	reqs, err := http.NewRequest("POST", uri, bytes.NewBuffer([]byte(jsonStr)))
+	if err != nil {
+		return err
+	}
+	reqs.Header.Add("Content-Type", "application/json")
+	reqs.Header.Add("Authorization", "Bearer "+token)
+	client := &http.Client{}
+	resp, err := client.Do(reqs)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}

report/stride_test.go

@@ -0,0 +1 @@
+package report

report/syslog.go

@@ -0,0 +1,111 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2018  Future Corporation , Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package report
+
+import (
+	"fmt"
+	"strings"
+
+	syslog "github.com/RackSec/srslog"
+
+	"github.com/pkg/errors"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+)
+
+// SyslogWriter send report to syslog
+type SyslogWriter struct{}
+
+func (w SyslogWriter) Write(rs ...models.ScanResult) (err error) {
+	conf := config.Conf.Syslog
+	facility, _ := conf.GetFacility()
+	severity, _ := conf.GetSeverity()
+	raddr := fmt.Sprintf("%s:%s", conf.Host, conf.Port)
+
+	sysLog, err := syslog.Dial(conf.Protocol, raddr, severity|facility, conf.Tag)
+	if err != nil {
+		return errors.Wrap(err, "Failed to initialize syslog client")
+	}
+
+	for _, r := range rs {
+		messages := w.encodeSyslog(r)
+		for _, m := range messages {
+			if _, err = fmt.Fprintf(sysLog, m); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (w SyslogWriter) encodeSyslog(result models.ScanResult) (messages []string) {
+	ipv4Addrs := strings.Join(result.IPv4Addrs, ",")
+	ipv6Addrs := strings.Join(result.IPv6Addrs, ",")
+
+	var commonKvPairs []string
+	commonKvPairs = append(commonKvPairs, fmt.Sprintf(`scanned_at="%s"`, result.ScannedAt))
+	commonKvPairs = append(commonKvPairs, fmt.Sprintf(`server_name="%s"`, result.ServerName))
+	commonKvPairs = append(commonKvPairs, fmt.Sprintf(`os_family="%s"`, result.Family))
+	commonKvPairs = append(commonKvPairs, fmt.Sprintf(`os_release="%s"`, result.Release))
+	commonKvPairs = append(commonKvPairs, fmt.Sprintf(`ipv4_addr="%s"`, ipv4Addrs))
+	commonKvPairs = append(commonKvPairs, fmt.Sprintf(`ipv6_addr="%s"`, ipv6Addrs))
+
+	for cveID, vinfo := range result.ScannedCves {
+		kvPairs := commonKvPairs
+
+		var pkgNames []string
+		for _, pkg := range vinfo.AffectedPackages {
+			pkgNames = append(pkgNames, pkg.Name)
+		}
+		pkgs := strings.Join(pkgNames, ",")
+		kvPairs = append(kvPairs, fmt.Sprintf(`packages="%s"`, pkgs))
+
+		kvPairs = append(kvPairs, fmt.Sprintf(`cve_id="%s"`, cveID))
+		for _, cvss := range vinfo.Cvss2Scores(result.Family) {
+			kvPairs = append(kvPairs, fmt.Sprintf(`cvss_score_%s_v2="%.2f"`, cvss.Type, cvss.Value.Score))
+			kvPairs = append(kvPairs, fmt.Sprintf(`cvss_vector_%s_v2="%s"`, cvss.Type, cvss.Value.Vector))
+		}
+
+		for _, cvss := range vinfo.Cvss3Scores() {
+			kvPairs = append(kvPairs, fmt.Sprintf(`cvss_score_%s_v3="%.2f"`, cvss.Type, cvss.Value.Score))
+			kvPairs = append(kvPairs, fmt.Sprintf(`cvss_vector_%s_v3="%s"`, cvss.Type, cvss.Value.Vector))
+		}
+
+		if content, ok := vinfo.CveContents[models.NvdXML]; ok {
+			cwes := strings.Join(content.CweIDs, ",")
+			kvPairs = append(kvPairs, fmt.Sprintf(`cwe_ids="%s"`, cwes))
+			if config.Conf.Syslog.Verbose {
+				kvPairs = append(kvPairs, fmt.Sprintf(`source_link="%s"`, content.SourceLink))
+				kvPairs = append(kvPairs, fmt.Sprintf(`summary="%s"`, content.Summary))
+			}
+		}
+		if content, ok := vinfo.CveContents[models.RedHat]; ok {
+			kvPairs = append(kvPairs, fmt.Sprintf(`title="%s"`, content.Title))
+		}
+
+		// message: key1="value1" key2="value2"...
+		messages = append(messages, strings.Join(kvPairs, " "))
+	}
+
+	if len(messages) == 0 {
+		commonKvPairs = append(commonKvPairs, `message="No CVE-IDs are found"`)
+		messages = append(messages, strings.Join(commonKvPairs, " "))
+	}
+	return messages
+}

report/syslog_test.go

@@ -0,0 +1,111 @@
+package report
+
+import (
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/future-architect/vuls/models"
+)
+
+func TestSyslogWriterEncodeSyslog(t *testing.T) {
+	var tests = []struct {
+		result           models.ScanResult
+		expectedMessages []string
+	}{
+		{
+			result: models.ScanResult{
+				ScannedAt:  time.Date(2018, 6, 13, 16, 10, 0, 0, time.UTC),
+				ServerName: "teste01",
+				Family:     "ubuntu",
+				Release:    "16.04",
+				IPv4Addrs:  []string{"192.168.0.1", "10.0.2.15"},
+				ScannedCves: models.VulnInfos{
+					"CVE-2017-0001": models.VulnInfo{
+						AffectedPackages: models.PackageStatuses{
+							models.PackageStatus{Name: "pkg1"},
+							models.PackageStatus{Name: "pkg2"},
+						},
+					},
+					"CVE-2017-0002": models.VulnInfo{
+						AffectedPackages: models.PackageStatuses{
+							models.PackageStatus{Name: "pkg3"},
+							models.PackageStatus{Name: "pkg4"},
+						},
+						CveContents: models.CveContents{
+							models.NvdXML: models.CveContent{
+								Cvss2Score:    5.0,
+								Cvss2Vector:   "AV:L/AC:L/Au:N/C:N/I:N/A:C",
+								Cvss2Severity: "MEDIUM",
+								CweIDs:        []string{"CWE-20"},
+							},
+						},
+					},
+				},
+			},
+			expectedMessages: []string{
+				`scanned_at="2018-06-13 16:10:00 +0000 UTC" server_name="teste01" os_family="ubuntu" os_release="16.04" ipv4_addr="192.168.0.1,10.0.2.15" ipv6_addr="" packages="pkg1,pkg2" cve_id="CVE-2017-0001"`,
+				`scanned_at="2018-06-13 16:10:00 +0000 UTC" server_name="teste01" os_family="ubuntu" os_release="16.04" ipv4_addr="192.168.0.1,10.0.2.15" ipv6_addr="" packages="pkg3,pkg4" cve_id="CVE-2017-0002" cvss_score_nvdxml_v2="5.00" cvss_vector_nvdxml_v2="AV:L/AC:L/Au:N/C:N/I:N/A:C" cwe_ids="CWE-20"`,
+			},
+		},
+		{
+			result: models.ScanResult{
+				ScannedAt:  time.Date(2018, 6, 13, 17, 10, 0, 0, time.UTC),
+				ServerName: "teste02",
+				Family:     "centos",
+				Release:    "6",
+				IPv6Addrs:  []string{"2001:0DB8::1"},
+				ScannedCves: models.VulnInfos{
+					"CVE-2017-0003": models.VulnInfo{
+						AffectedPackages: models.PackageStatuses{
+							models.PackageStatus{Name: "pkg5"},
+						},
+						CveContents: models.CveContents{
+							models.RedHat: models.CveContent{
+								Cvss3Score:  5.0,
+								Cvss3Vector: "AV:L/AC:L/Au:N/C:N/I:N/A:C",
+								CweIDs:      []string{"CWE-284"},
+								Title:       "RHSA-2017:0001: pkg5 security update (Important)",
+							},
+						},
+					},
+				},
+			},
+			expectedMessages: []string{
+				`scanned_at="2018-06-13 17:10:00 +0000 UTC" server_name="teste02" os_family="centos" os_release="6" ipv4_addr="" ipv6_addr="2001:0DB8::1" packages="pkg5" cve_id="CVE-2017-0003" cvss_score_redhat_v3="5.00" cvss_vector_redhat_v3="AV:L/AC:L/Au:N/C:N/I:N/A:C" title="RHSA-2017:0001: pkg5 security update (Important)"`,
+			},
+		},
+		{
+			result: models.ScanResult{
+				ScannedAt:   time.Date(2018, 6, 13, 12, 10, 0, 0, time.UTC),
+				ServerName:  "teste03",
+				Family:      "centos",
+				Release:     "7",
+				IPv6Addrs:   []string{"2001:0DB8::1"},
+				ScannedCves: models.VulnInfos{},
+			},
+			expectedMessages: []string{
+				`scanned_at="2018-06-13 12:10:00 +0000 UTC" server_name="teste03" os_family="centos" os_release="7" ipv4_addr="" ipv6_addr="2001:0DB8::1" message="No CVE-IDs are found"`,
+			},
+		},
+	}
+
+	for i, tt := range tests {
+		messages := SyslogWriter{}.encodeSyslog(tt.result)
+		if len(messages) != len(tt.expectedMessages) {
+			t.Fatalf("test: %d, Message Length: expected %d, actual: %d",
+				i, len(tt.expectedMessages), len(messages))
+		}
+
+		sort.Slice(messages, func(i, j int) bool {
+			return messages[i] < messages[j]
+		})
+
+		for j, m := range messages {
+			e := tt.expectedMessages[j]
+			if e != m {
+				t.Errorf("test: %d, Messsage %d: \nexpected %s \nactual   %s", i, j, e, m)
+			}
+		}
+	}
+}

report/tui.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -32,7 +32,6 @@ import (
 	"github.com/google/subcommands"
 	"github.com/gosuri/uitable"
 	"github.com/jroimartin/gocui"
-	log "github.com/sirupsen/logrus"
 )
 
 var scanResults models.ScanResults
@@ -52,18 +51,16 @@ func RunTui(results models.ScanResults) subcommands.ExitStatus {
 		return scanResults[i].ServerName < scanResults[j].ServerName
 	})
 
-	// g, err := gocui.NewGui(gocui.OutputNormal)
-	g := gocui.NewGui()
-	if err := g.Init(); err != nil {
-		log.Errorf("%s", err)
+	g, err := gocui.NewGui(gocui.OutputNormal)
+	if err != nil {
+		util.Log.Errorf("%s", err)
 		return subcommands.ExitFailure
 	}
 	defer g.Close()
 
-	g.SetLayout(layout)
-	// g.SetManagerFunc(layout)
+	g.SetManagerFunc(layout)
 	if err := keybindings(g); err != nil {
-		log.Errorf("%s", err)
+		util.Log.Errorf("%s", err)
 		return subcommands.ExitFailure
 	}
 	g.SelBgColor = gocui.ColorGreen
@@ -72,7 +69,7 @@ func RunTui(results models.ScanResults) subcommands.ExitStatus {
 
 	if err := g.MainLoop(); err != nil {
 		g.Close()
-		log.Errorf("%s", err)
+		util.Log.Errorf("%s", err)
 		os.Exit(1)
 	}
 	return subcommands.ExitSuccess
@@ -186,19 +183,19 @@ func nextView(g *gocui.Gui, v *gocui.View) error {
 	var err error
 
 	if v == nil {
-		err = g.SetCurrentView("side")
+		_, err = g.SetCurrentView("side")
 	}
 	switch v.Name() {
 	case "side":
-		err = g.SetCurrentView("summary")
+		_, err = g.SetCurrentView("summary")
 	case "summary":
-		err = g.SetCurrentView("detail")
+		_, err = g.SetCurrentView("detail")
 	case "detail":
-		err = g.SetCurrentView("changelog")
+		_, err = g.SetCurrentView("changelog")
 	case "changelog":
-		err = g.SetCurrentView("side")
+		_, err = g.SetCurrentView("side")
 	default:
-		err = g.SetCurrentView("summary")
+		_, err = g.SetCurrentView("summary")
 	}
 	return err
 }
@@ -207,19 +204,19 @@ func previousView(g *gocui.Gui, v *gocui.View) error {
 	var err error
 
 	if v == nil {
-		err = g.SetCurrentView("side")
+		_, err = g.SetCurrentView("side")
 	}
 	switch v.Name() {
 	case "side":
-		err = g.SetCurrentView("side")
+		_, err = g.SetCurrentView("side")
 	case "summary":
-		err = g.SetCurrentView("side")
+		_, err = g.SetCurrentView("side")
 	case "detail":
-		err = g.SetCurrentView("summary")
+		_, err = g.SetCurrentView("summary")
 	case "changelog":
-		err = g.SetCurrentView("detail")
+		_, err = g.SetCurrentView("detail")
 	default:
-		err = g.SetCurrentView("side")
+		_, err = g.SetCurrentView("side")
 	}
 	return err
 }
@@ -402,7 +399,7 @@ func cursorPageUp(g *gocui.Gui, v *gocui.View) error {
 func previousSummary(g *gocui.Gui, v *gocui.View) error {
 	if v != nil {
 		// cursor to summary
-		if err := g.SetCurrentView("summary"); err != nil {
+		if _, err := g.SetCurrentView("summary"); err != nil {
 			return err
 		}
 		// move next line
@@ -410,7 +407,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error {
 			return err
 		}
 		// cursor to detail
-		if err := g.SetCurrentView("detail"); err != nil {
+		if _, err := g.SetCurrentView("detail"); err != nil {
 			return err
 		}
 	}
@@ -420,7 +417,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error {
 func nextSummary(g *gocui.Gui, v *gocui.View) error {
 	if v != nil {
 		// cursor to summary
-		if err := g.SetCurrentView("summary"); err != nil {
+		if _, err := g.SetCurrentView("summary"); err != nil {
 			return err
 		}
 		// move next line
@@ -428,7 +425,7 @@ func nextSummary(g *gocui.Gui, v *gocui.View) error {
 			return err
 		}
 		// cursor to detail
-		if err := g.SetCurrentView("detail"); err != nil {
+		if _, err := g.SetCurrentView("detail"); err != nil {
 			return err
 		}
 	}
@@ -468,10 +465,7 @@ func changeHost(g *gocui.Gui, v *gocui.View) error {
 	if err := setDetailLayout(g); err != nil {
 		return err
 	}
-	if err := setChangelogLayout(g); err != nil {
-		return err
-	}
-	return nil
+	return setChangelogLayout(g)
 }
 
 func redrawDetail(g *gocui.Gui) error {
@@ -479,10 +473,7 @@ func redrawDetail(g *gocui.Gui) error {
 		return err
 	}
 
-	if err := setDetailLayout(g); err != nil {
-		return err
-	}
-	return nil
+	return setDetailLayout(g)
 }
 
 func redrawChangelog(g *gocui.Gui) error {
@@ -490,10 +481,7 @@ func redrawChangelog(g *gocui.Gui) error {
 		return err
 	}
 
-	if err := setChangelogLayout(g); err != nil {
-		return err
-	}
-	return nil
+	return setChangelogLayout(g)
 }
 
 func getLine(g *gocui.Gui, v *gocui.View) error {
@@ -511,7 +499,7 @@ func getLine(g *gocui.Gui, v *gocui.View) error {
 			return err
 		}
 		fmt.Fprintln(v, l)
-		if err := g.SetCurrentView("msg"); err != nil {
+		if _, err := g.SetCurrentView("msg"); err != nil {
 			return err
 		}
 	}
@@ -534,7 +522,7 @@ func showMsg(g *gocui.Gui, v *gocui.View) error {
 			return err
 		}
 		fmt.Fprintln(v, l)
-		if err := g.SetCurrentView("msg"); err != nil {
+		if _, err := g.SetCurrentView("msg"); err != nil {
 			return err
 		}
 	}
@@ -545,10 +533,8 @@ func delMsg(g *gocui.Gui, v *gocui.View) error {
 	if err := g.DeleteView("msg"); err != nil {
 		return err
 	}
-	if err := g.SetCurrentView("summary"); err != nil {
-		return err
-	}
-	return nil
+	_, err := g.SetCurrentView("summary")
+	return err
 }
 
 func quit(g *gocui.Gui, v *gocui.View) error {
@@ -565,11 +551,7 @@ func layout(g *gocui.Gui) error {
 	if err := setDetailLayout(g); err != nil {
 		return err
 	}
-	if err := setChangelogLayout(g); err != nil {
-		return err
-	}
-
-	return nil
+	return setChangelogLayout(g)
 }
 
 func debug(g *gocui.Gui, str string) error {
@@ -601,7 +583,7 @@ func setSideLayout(g *gocui.Gui) error {
 		}
 		currentScanResult = scanResults[0]
 		vinfos = scanResults[0].ScannedCves.ToSortedSlice()
-		if err := g.SetCurrentView("side"); err != nil {
+		if _, err := g.SetCurrentView("side"); err != nil {
 			return err
 		}
 	}
@@ -615,7 +597,7 @@ func setSummaryLayout(g *gocui.Gui) error {
 			return err
 		}
 
-		lines := summaryLines()
+		lines := summaryLines(currentScanResult)
 		fmt.Fprintf(v, lines)
 
 		v.Highlight = true
@@ -625,37 +607,42 @@ func setSummaryLayout(g *gocui.Gui) error {
 	return nil
 }
 
-func summaryLines() string {
+func summaryLines(r models.ScanResult) string {
 	stable := uitable.New()
 	stable.MaxColWidth = 1000
 	stable.Wrap = false
 
-	if len(currentScanResult.Errors) != 0 {
+	if len(r.Errors) != 0 {
 		return "Error: Scan with --debug to view the details"
 	}
 
 	indexFormat := ""
-	if len(currentScanResult.ScannedCves) < 10 {
+	if len(r.ScannedCves) < 10 {
 		indexFormat = "[%1d]"
-	} else if len(currentScanResult.ScannedCves) < 100 {
+	} else if len(r.ScannedCves) < 100 {
 		indexFormat = "[%2d]"
 	} else {
 		indexFormat = "[%3d]"
 	}
 
-	for i, vinfo := range vinfos {
-		summary := vinfo.Titles(
-			config.Conf.Lang, currentScanResult.Family)[0].Value
-		cvssScore := fmt.Sprintf("| %4.1f",
-			vinfo.MaxCvssScore().Value.Score)
+	for i, vinfo := range r.ScannedCves.ToSortedSlice() {
+		max := vinfo.MaxCvssScore().Value.Score
+		cvssScore := "|     "
+		if 0 < max {
+			cvssScore = fmt.Sprintf("| %4.1f", max)
+		}
+
+		packname := vinfo.AffectedPackages.FormatTuiSummary()
+		packname += strings.Join(vinfo.CpeURIs, ", ")
 
 		var cols []string
 		cols = []string{
 			fmt.Sprintf(indexFormat, i+1),
 			vinfo.CveID,
-			cvssScore,
-			fmt.Sprintf("| %3d |", vinfo.Confidence.Score),
-			summary,
+			cvssScore + " |",
+			fmt.Sprintf("%8s |", vinfo.AttackVector()),
+			fmt.Sprintf("%7s |", vinfo.PatchStatus(r.Packages)),
+			packname,
 		}
 		icols := make([]interface{}, len(cols))
 		for j := range cols {
@@ -695,16 +682,12 @@ func setDetailLayout(g *gocui.Gui) error {
 }
 
 func setChangelogLayout(g *gocui.Gui) error {
-	maxX, maxY := g.Size()
-
 	summaryView, err := g.View("summary")
 	if err != nil {
 		return err
 	}
-	_, cy := summaryView.Cursor()
-	_, oy := summaryView.Origin()
-	currentVinfo = cy + oy
 
+	maxX, maxY := g.Size()
 	if v, err := g.SetView("changelog", int(float64(maxX)*0.5), int(float64(maxY)*0.2), maxX, maxY); err != nil {
 		if err != gocui.ErrUnknownView {
 			return err
@@ -713,17 +696,74 @@ func setChangelogLayout(g *gocui.Gui) error {
 			return nil
 		}
 
-		lines := []string{}
+		lines := []string{
+			"Affected Packages, Processes",
+			"============================",
+		}
+
+		_, cy := summaryView.Cursor()
+		_, oy := summaryView.Origin()
+		currentVinfo = cy + oy
 		vinfo := vinfos[currentVinfo]
+		vinfo.AffectedPackages.Sort()
+		for _, affected := range vinfo.AffectedPackages {
+			// packages detected by OVAL may not be actually installed
+			if pack, ok := currentScanResult.Packages[affected.Name]; ok {
+				var line string
+				if pack.Repository != "" {
+					line = fmt.Sprintf("* %s (%s)",
+						pack.FormatVersionFromTo(affected.NotFixedYet, affected.FixState),
+						pack.Repository)
+				} else {
+					line = fmt.Sprintf("* %s",
+						pack.FormatVersionFromTo(affected.NotFixedYet, affected.FixState),
+					)
+				}
+				lines = append(lines, line)
+
+				if len(pack.AffectedProcs) != 0 {
+					for _, p := range pack.AffectedProcs {
+						lines = append(lines, fmt.Sprintf("  * PID: %s %s", p.PID, p.Name))
+					}
+				} else {
+					// lines = append(lines, fmt.Sprintf("  * No affected process"))
+				}
+			}
+		}
+		sort.Strings(vinfo.CpeURIs)
+		for _, uri := range vinfo.CpeURIs {
+			lines = append(lines, "* "+uri)
+		}
+
 		for _, adv := range vinfo.DistroAdvisories {
+			lines = append(lines, "\n",
+				"Advisories",
+				"==========",
+			)
 			lines = append(lines, adv.Format())
 		}
 
-		for _, affected := range vinfo.AffectedPackages {
-			pack := currentScanResult.Packages[affected.Name]
-			for _, p := range currentScanResult.Packages {
-				if pack.Name == p.Name {
-					lines = append(lines, p.FormatChangelog(), "\n")
+		if len(vinfo.Exploits) != 0 {
+			lines = append(lines, "\n",
+				"Exploit Codes",
+				"=============",
+			)
+			for _, exploit := range vinfo.Exploits {
+				lines = append(lines, fmt.Sprintf("* [%s](%s)", exploit.Description, exploit.URL))
+			}
+		}
+
+		if currentScanResult.IsDeepScanMode() {
+			lines = append(lines, "\n",
+				"ChangeLogs",
+				"==========",
+			)
+			for _, affected := range vinfo.AffectedPackages {
+				pack := currentScanResult.Packages[affected.Name]
+				for _, p := range currentScanResult.Packages {
+					if pack.Name == p.Name {
+						lines = append(lines, p.FormatChangelog(), "\n")
+					}
 				}
 			}
 		}
@@ -740,13 +780,15 @@ func setChangelogLayout(g *gocui.Gui) error {
 type dataForTmpl struct {
 	CveID            string
 	Cvsses           string
+	Exploits         []models.Exploit
 	Summary          string
-	Confidence       models.Confidence
-	Cwes             []models.CveContentStr
+	Mitigation       string
+	Confidences      models.Confidences
+	Cwes             []models.CweDictEntry
 	Links            []string
 	References       []models.Reference
 	Packages         []string
-	CpeNames         []string
+	CpeURIs          []string
 	PublishedDate    time.Time
 	LastModifiedDate time.Time
 }
@@ -767,20 +809,6 @@ func detailLines() (string, error) {
 	}
 
 	vinfo := vinfos[currentVinfo]
-
-	packsVer := []string{}
-	vinfo.AffectedPackages.Sort()
-	for _, affected := range vinfo.AffectedPackages {
-		// packages detected by OVAL may not be actually installed
-		if pack, ok := r.Packages[affected.Name]; ok {
-			packsVer = append(packsVer, pack.FormatVersionFromTo(affected.NotFixedYet))
-		}
-	}
-	sort.Strings(vinfo.CpeNames)
-	for _, name := range vinfo.CpeNames {
-		packsVer = append(packsVer, name)
-	}
-
 	links := []string{vinfo.CveContents.SourceLinks(
 		config.Conf.Lang, r.Family, vinfo.CveID)[0].Value,
 		vinfo.Cvss2CalcURL(),
@@ -792,35 +820,57 @@ func detailLines() (string, error) {
 	refs := []models.Reference{}
 	for _, rr := range vinfo.CveContents.References(r.Family) {
 		for _, ref := range rr.Value {
+			if ref.Source == "" {
+				ref.Source = "-"
+			}
 			refs = append(refs, ref)
 		}
 	}
 
 	summary := vinfo.Summaries(r.Lang, r.Family)[0]
+	mitigation := vinfo.Mitigations(r.Family)[0]
 
 	table := uitable.New()
 	table.MaxColWidth = maxColWidth
 	table.Wrap = true
-	scores := append(vinfo.Cvss3Scores(), vinfo.Cvss2Scores()...)
+	scores := append(vinfo.Cvss3Scores(), vinfo.Cvss2Scores(r.Family)...)
 	var cols []interface{}
 	for _, score := range scores {
+		if score.Value.Score == 0 && score.Value.Severity == "" {
+			continue
+		}
+		scoreStr := "-"
+		if 0 < score.Value.Score {
+			scoreStr = fmt.Sprintf("%3.1f", score.Value.Score)
+		}
+		scoreVec := fmt.Sprintf("%s/%s", scoreStr, score.Value.Vector)
 		cols = []interface{}{
+			scoreVec,
 			score.Value.Severity,
-			score.Value.Format(),
 			score.Type,
 		}
 		table.AddRow(cols...)
 	}
 
+	uniqCweIDs := vinfo.CveContents.UniqCweIDs(r.Family)
+	cwes := []models.CweDictEntry{}
+	for _, cweID := range uniqCweIDs {
+		if strings.HasPrefix(cweID.Value, "CWE-") {
+			if dict, ok := r.CweDict[strings.TrimPrefix(cweID.Value, "CWE-")]; ok {
+				cwes = append(cwes, dict)
+			}
+		}
+	}
+
 	data := dataForTmpl{
-		CveID:      vinfo.CveID,
-		Cvsses:     fmt.Sprintf("%s\n", table),
-		Summary:    fmt.Sprintf("%s (%s)", summary.Value, summary.Type),
-		Confidence: vinfo.Confidence,
-		Cwes:       vinfo.CveContents.CweIDs(r.Family),
-		Links:      util.Distinct(links),
-		Packages:   packsVer,
-		References: refs,
+		CveID:       vinfo.CveID,
+		Cvsses:      fmt.Sprintf("%s\n", table),
+		Summary:     fmt.Sprintf("%s (%s)", summary.Value, summary.Type),
+		Mitigation:  fmt.Sprintf("%s (%s)", mitigation.Value, mitigation.Type),
+		Confidences: vinfo.Confidences,
+		Cwes:        cwes,
+		Links:       util.Distinct(links),
+		References:  refs,
 	}
 
 	buf := bytes.NewBuffer(nil) // create empty buffer
@@ -833,47 +883,42 @@ func detailLines() (string, error) {
 
 const mdTemplate = `
 {{.CveID}}
-==============
+================
 
 CVSS Scores
---------------
+-----------
 {{.Cvsses }}
 
 Summary
---------------
+-----------
  {{.Summary }}
 
+Mitigation
+-----------
+ {{.Mitigation }}
 
 Links
---------------
+-----------
 {{range $link := .Links -}}
 * {{$link}}
 {{end}}
-
 CWE
---------------
+-----------
 {{range .Cwes -}}
-* {{.Value}} ({{.Type}})
+* {{.En.CweID}} [{{.En.Name}}](https://cwe.mitre.org/data/definitions/{{.En.CweID}}.html)
 {{end}}
-
-Package/CPE
---------------
-{{range $pack := .Packages -}}
-* {{$pack}}
-{{end -}}
-{{range $name := .CpeNames -}}
+{{range $name := .CpeURIs -}}
 * {{$name}}
 {{end}}
-
 Confidence
---------------
- {{.Confidence }}
-
-
+-----------
+{{range $confidence := .Confidences -}}
+* {{$confidence.DetectionMethod}}
+{{end}}
 References
---------------
+-----------
 {{range .References -}}
-* [{{.Source}}]( {{.Link}} )
+* [{{.Source}}]({{.Link}})
 {{end}}
 
 `

report/util.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,11 +18,13 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package report
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"sort"
 	"strings"
@@ -32,9 +34,10 @@ import (
 	"github.com/future-architect/vuls/models"
 	"github.com/future-architect/vuls/util"
 	"github.com/gosuri/uitable"
+	"github.com/olekukonko/tablewriter"
 )
 
-const maxColWidth = 80
+const maxColWidth = 100
 
 func formatScanSummary(rs ...models.ScanResult) string {
 	table := uitable.New()
@@ -46,7 +49,8 @@ func formatScanSummary(rs ...models.ScanResult) string {
 			cols = []interface{}{
 				r.FormatServerName(),
 				fmt.Sprintf("%s%s", r.Family, r.Release),
-				r.Packages.FormatUpdatablePacksSummary(),
+				r.FormatUpdatablePacksSummary(),
+				r.FormatExploitCveSummary(),
 			}
 		} else {
 			cols = []interface{}{
@@ -71,7 +75,9 @@ func formatOneLineSummary(rs ...models.ScanResult) string {
 			cols = []interface{}{
 				r.FormatServerName(),
 				r.ScannedCves.FormatCveSummary(),
-				r.Packages.FormatUpdatablePacksSummary(),
+				r.ScannedCves.FormatFixedStatus(r.Packages),
+				r.FormatUpdatablePacksSummary(),
+				r.FormatExploitCveSummary(),
 			}
 		} else {
 			cols = []interface{}{
@@ -85,7 +91,7 @@ func formatOneLineSummary(rs ...models.ScanResult) string {
 	return fmt.Sprintf("%s\n", table)
 }
 
-func formatShortPlainText(r models.ScanResult) string {
+func formatList(r models.ScanResult) string {
 	header := r.FormatTextReportHeadedr()
 	if len(r.Errors) != 0 {
 		return fmt.Sprintf(
@@ -98,63 +104,51 @@ func formatShortPlainText(r models.ScanResult) string {
 %s
 No CVE-IDs are found in updatable packages.
 %s
-	 `, header, r.Packages.FormatUpdatablePacksSummary())
+	 `, header, r.FormatUpdatablePacksSummary())
 	}
 
-	stable := uitable.New()
-	stable.MaxColWidth = maxColWidth
-	stable.Wrap = true
-	for _, vuln := range r.ScannedCves.ToSortedSlice() {
-		summaries := vuln.Summaries(config.Conf.Lang, r.Family)
-		links := vuln.CveContents.SourceLinks(
-			config.Conf.Lang, r.Family, vuln.CveID)
+	data := [][]string{}
+	for _, vinfo := range r.ScannedCves.ToSortedSlice() {
+		max := vinfo.MaxCvssScore().Value.Score
+		// v2max := vinfo.MaxCvss2Score().Value.Score
+		// v3max := vinfo.MaxCvss3Score().Value.Score
 
-		vlinks := []string{}
-		for name, url := range vuln.VendorLinks(r.Family) {
-			vlinks = append(vlinks, fmt.Sprintf("%s (%s)", url, name))
-		}
+		// packname := vinfo.AffectedPackages.FormatTuiSummary()
+		// packname += strings.Join(vinfo.CpeURIs, ", ")
 
-		cvsses := ""
-		for _, cvss := range vuln.Cvss2Scores() {
-			cvsses += fmt.Sprintf("%s (%s)\n", cvss.Value.Format(), cvss.Type)
-		}
-		cvsses += vuln.Cvss2CalcURL() + "\n"
-		for _, cvss := range vuln.Cvss3Scores() {
-			cvsses += fmt.Sprintf("%s (%s)\n", cvss.Value.Format(), cvss.Type)
-		}
-		if 0 < len(vuln.Cvss3Scores()) {
-			cvsses += vuln.Cvss3CalcURL() + "\n"
-		}
-
-		maxCvss := vuln.FormatMaxCvssScore()
-		rightCol := fmt.Sprintf(`%s
-%s
----
-%s
-%s
-%sConfidence: %v`,
-			maxCvss,
-			summaries[0].Value,
-			links[0].Value,
-			strings.Join(vlinks, "\n"),
-			cvsses,
-			//  packsVer,
-			vuln.Confidence,
-		)
-
-		leftCol := fmt.Sprintf("%s", vuln.CveID)
-		scols := []string{leftCol, rightCol}
-		cols := make([]interface{}, len(scols))
-		for i := range cols {
-			cols[i] = scols[i]
-		}
-		stable.AddRow(cols...)
-		stable.AddRow("")
+		data = append(data, []string{
+			vinfo.CveID,
+			fmt.Sprintf("%4.1f", max),
+			// fmt.Sprintf("%4.1f", v2max),
+			// fmt.Sprintf("%4.1f", v3max),
+			fmt.Sprintf("%8s", vinfo.AttackVector()),
+			fmt.Sprintf("%7s", vinfo.PatchStatus(r.Packages)),
+			// packname,
+			fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", vinfo.CveID),
+			fmt.Sprintf("%t", 0 < len(vinfo.Exploits)),
+		})
 	}
-	return fmt.Sprintf("%s\n%s\n", header, stable)
+
+	b := bytes.Buffer{}
+	table := tablewriter.NewWriter(&b)
+	table.SetHeader([]string{
+		"CVE-ID",
+		"CVSS",
+		// "v3",
+		// "v2",
+		"Attack",
+		"Fixed",
+		// "Pkg",
+		"NVD",
+		"Exploit",
+	})
+	table.SetBorder(true)
+	table.AppendBulk(data)
+	table.Render()
+	return fmt.Sprintf("%s\n%s", header, b.String())
 }
 
-func formatFullPlainText(r models.ScanResult) string {
+func formatFullPlainText(r models.ScanResult) (lines string) {
 	header := r.FormatTextReportHeadedr()
 	if len(r.Errors) != 0 {
 		return fmt.Sprintf(
@@ -167,62 +161,126 @@ func formatFullPlainText(r models.ScanResult) string {
 %s
 No CVE-IDs are found in updatable packages.
 %s
-	 `, header, r.Packages.FormatUpdatablePacksSummary())
+	 `, header, r.FormatUpdatablePacksSummary())
 	}
 
-	table := uitable.New()
-	table.MaxColWidth = maxColWidth
-	table.Wrap = true
+	lines = header + "\n"
+
 	for _, vuln := range r.ScannedCves.ToSortedSlice() {
-		table.AddRow(vuln.CveID)
-		table.AddRow("----------------")
-		table.AddRow("Max Score", vuln.FormatMaxCvssScore())
-		for _, cvss := range vuln.Cvss2Scores() {
-			table.AddRow(cvss.Type, cvss.Value.Format())
-		}
+		data := [][]string{}
+		data = append(data, []string{"Max Score", vuln.FormatMaxCvssScore()})
 		for _, cvss := range vuln.Cvss3Scores() {
-			table.AddRow(cvss.Type, cvss.Value.Format())
-		}
-		if 0 < len(vuln.Cvss2Scores()) {
-			table.AddRow("CVSSv2 Calc", vuln.Cvss2CalcURL())
-		}
-		if 0 < len(vuln.Cvss3Scores()) {
-			table.AddRow("CVSSv3 Calc", vuln.Cvss3CalcURL())
-		}
-		table.AddRow("Summary", vuln.Summaries(
-			config.Conf.Lang, r.Family)[0].Value)
-
-		links := vuln.CveContents.SourceLinks(
-			config.Conf.Lang, r.Family, vuln.CveID)
-		table.AddRow("Source", links[0].Value)
-
-		vlinks := vuln.VendorLinks(r.Family)
-		for name, url := range vlinks {
-			table.AddRow(name, url)
+			if cvssstr := cvss.Value.Format(); cvssstr != "" {
+				data = append(data, []string{string(cvss.Type), cvssstr})
+			}
 		}
 
-		for _, v := range vuln.CveContents.CweIDs(r.Family) {
-			table.AddRow(fmt.Sprintf("%s (%s)", v.Value, v.Type), cweURL(v.Value))
+		for _, cvss := range vuln.Cvss2Scores(r.Family) {
+			if cvssstr := cvss.Value.Format(); cvssstr != "" {
+				data = append(data, []string{string(cvss.Type), cvssstr})
+			}
+		}
+
+		data = append(data, []string{"Summary", vuln.Summaries(
+			config.Conf.Lang, r.Family)[0].Value})
+
+		mitigation := vuln.Mitigations(r.Family)[0]
+		if mitigation.Type != models.Unknown {
+			data = append(data, []string{"Mitigation", mitigation.Value})
+		}
+
+		cweURLs, top10URLs := []string{}, []string{}
+		for _, v := range vuln.CveContents.UniqCweIDs(r.Family) {
+			name, url, top10Rank, top10URL := r.CweDict.Get(v.Value, r.Lang)
+			if top10Rank != "" {
+				data = append(data, []string{"CWE",
+					fmt.Sprintf("[OWASP Top%s] %s: %s (%s)",
+						top10Rank, v.Value, name, v.Type)})
+				top10URLs = append(top10URLs, top10URL)
+			} else {
+				data = append(data, []string{"CWE", fmt.Sprintf("%s: %s (%s)",
+					v.Value, name, v.Type)})
+			}
+			cweURLs = append(cweURLs, url)
 		}
 
-		packsVer := []string{}
 		vuln.AffectedPackages.Sort()
 		for _, affected := range vuln.AffectedPackages {
 			if pack, ok := r.Packages[affected.Name]; ok {
-				packsVer = append(packsVer, pack.FormatVersionFromTo(affected.NotFixedYet))
+				var line string
+				if pack.Repository != "" {
+					line = fmt.Sprintf("%s (%s)",
+						pack.FormatVersionFromTo(affected.NotFixedYet, affected.FixState),
+						pack.Repository)
+				} else {
+					line = fmt.Sprintf("%s",
+						pack.FormatVersionFromTo(affected.NotFixedYet, affected.FixState),
+					)
+				}
+				data = append(data, []string{"Affected Pkg", line})
+
+				if len(pack.AffectedProcs) != 0 {
+					for _, p := range pack.AffectedProcs {
+						data = append(data, []string{"",
+							fmt.Sprintf("  - PID: %s %s", p.PID, p.Name)})
+					}
+				}
 			}
 		}
-		sort.Strings(vuln.CpeNames)
-		for _, name := range vuln.CpeNames {
-			packsVer = append(packsVer, name)
+		sort.Strings(vuln.CpeURIs)
+		for _, name := range vuln.CpeURIs {
+			data = append(data, []string{"CPE", name})
 		}
-		table.AddRow("Package/CPE", strings.Join(packsVer, "\n"))
-		table.AddRow("Confidence", vuln.Confidence)
 
-		table.AddRow("\n")
+		for _, confidence := range vuln.Confidences {
+			data = append(data, []string{"Confidence", confidence.String()})
+		}
+
+		links := vuln.CveContents.SourceLinks(
+			config.Conf.Lang, r.Family, vuln.CveID)
+		data = append(data, []string{"Source", links[0].Value})
+
+		if 0 < len(vuln.Cvss2Scores(r.Family)) {
+			data = append(data, []string{"CVSSv2 Calc", vuln.Cvss2CalcURL()})
+		}
+		if 0 < len(vuln.Cvss3Scores()) {
+			data = append(data, []string{"CVSSv3 Calc", vuln.Cvss3CalcURL()})
+		}
+
+		vlinks := vuln.VendorLinks(r.Family)
+		for name, url := range vlinks {
+			data = append(data, []string{name, url})
+		}
+		for _, url := range cweURLs {
+			data = append(data, []string{"CWE", url})
+		}
+		for _, exploit := range vuln.Exploits {
+			data = append(data, []string{string(exploit.ExploitType), exploit.URL})
+		}
+		for _, url := range top10URLs {
+			data = append(data, []string{"OWASP Top10", url})
+		}
+
+		// for _, rr := range vuln.CveContents.References(r.Family) {
+		// for _, ref := range rr.Value {
+		// data = append(data, []string{ref.Source, ref.Link})
+		// }
+		// }
+
+		b := bytes.Buffer{}
+		table := tablewriter.NewWriter(&b)
+		table.SetColWidth(80)
+		table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+		table.SetHeader([]string{
+			vuln.CveID,
+			"",
+		})
+		table.SetBorder(true)
+		table.AppendBulk(data)
+		table.Render()
+		lines += b.String() + "\n"
 	}
-
-	return fmt.Sprintf("%s\n%s", header, table)
+	return
 }
 
 func cweURL(cweID string) string {
@@ -273,27 +331,35 @@ func overwriteJSONFile(dir string, r models.ScanResult) error {
 	return nil
 }
 
-func loadPrevious(current models.ScanResults) (previous models.ScanResults, err error) {
+func loadPrevious(currs models.ScanResults) (prevs models.ScanResults, err error) {
 	dirs, err := ListValidJSONDirs()
 	if err != nil {
 		return
 	}
 
-	for _, result := range current {
+	for _, result := range currs {
+		filename := result.ServerName + ".json"
+		if result.Container.Name != "" {
+			filename = fmt.Sprintf("%s@%s.json", result.Container.Name, result.ServerName)
+		}
 		for _, dir := range dirs[1:] {
-			var r *models.ScanResult
-			path := filepath.Join(dir, result.ServerName+".json")
-			if r, err = loadOneServerScanResult(path); err != nil {
+			path := filepath.Join(dir, filename)
+			r, err := loadOneServerScanResult(path)
+			if err != nil {
+				util.Log.Errorf("%s", err)
 				continue
 			}
 			if r.Family == result.Family && r.Release == result.Release {
-				previous = append(previous, *r)
+				prevs = append(prevs, *r)
 				util.Log.Infof("Previous json found: %s", path)
 				break
+			} else {
+				util.Log.Infof("Previous json is different family.Release: %s, pre: %s.%s cur: %s.%s",
+					path, r.Family, r.Release, result.Family, result.Release)
 			}
 		}
 	}
-	return previous, nil
+	return prevs, nil
 }
 
 func diff(curResults, preResults models.ScanResults) (diffed models.ScanResults, err error) {
@@ -301,7 +367,7 @@ func diff(curResults, preResults models.ScanResults) (diffed models.ScanResults,
 		found := false
 		var previous models.ScanResult
 		for _, r := range preResults {
-			if current.ServerName == r.ServerName {
+			if current.ServerName == r.ServerName && current.Container.Name == r.Container.Name {
 				found = true
 				previous = r
 				break
@@ -337,8 +403,20 @@ func getDiffCves(previous, current models.ScanResult) models.VulnInfos {
 		if previousCveIDsSet[v.CveID] {
 			if isCveInfoUpdated(v.CveID, previous, current) {
 				updated[v.CveID] = v
+				util.Log.Debugf("updated: %s", v.CveID)
+
+				// TODO commented out beause  a bug of diff logic when multiple oval defs found for a certain CVE-ID and same updated_at
+				// if these OVAL defs have different affected packages, this logic detects as updated.
+				// This logic will be uncommented after integration with ghost https://github.com/knqyf263/gost
+				// } else if isCveFixed(v, previous) {
+				// updated[v.CveID] = v
+				// util.Log.Debugf("fixed: %s", v.CveID)
+
+			} else {
+				util.Log.Debugf("same: %s", v.CveID)
 			}
 		} else {
+			util.Log.Debugf("new: %s", v.CveID)
 			new[v.CveID] = v
 		}
 	}
@@ -349,36 +427,54 @@ func getDiffCves(previous, current models.ScanResult) models.VulnInfos {
 	return updated
 }
 
+func isCveFixed(current models.VulnInfo, previous models.ScanResult) bool {
+	preVinfo, _ := previous.ScannedCves[current.CveID]
+	pre := map[string]bool{}
+	for _, h := range preVinfo.AffectedPackages {
+		pre[h.Name] = h.NotFixedYet
+	}
+
+	cur := map[string]bool{}
+	for _, h := range current.AffectedPackages {
+		cur[h.Name] = h.NotFixedYet
+	}
+
+	return !reflect.DeepEqual(pre, cur)
+}
+
 func isCveInfoUpdated(cveID string, previous, current models.ScanResult) bool {
 	cTypes := []models.CveContentType{
-		models.NVD,
-		models.JVN,
+		models.NvdXML,
+		models.Jvn,
 		models.NewCveContentType(current.Family),
 	}
 
 	prevLastModified := map[models.CveContentType]time.Time{}
-	for _, c := range previous.ScannedCves {
-		if cveID == c.CveID {
-			for _, cType := range cTypes {
-				content, _ := c.CveContents[cType]
-				prevLastModified[cType] = content.LastModified
-			}
-			break
+	preVinfo, ok := previous.ScannedCves[cveID]
+	if !ok {
+		return true
+	}
+	for _, cType := range cTypes {
+		if content, ok := preVinfo.CveContents[cType]; ok {
+			prevLastModified[cType] = content.LastModified
 		}
 	}
 
 	curLastModified := map[models.CveContentType]time.Time{}
-	for _, c := range current.ScannedCves {
-		if cveID == c.CveID {
-			for _, cType := range cTypes {
-				content, _ := c.CveContents[cType]
-				curLastModified[cType] = content.LastModified
-			}
-			break
-		}
+	curVinfo, ok := current.ScannedCves[cveID]
+	if !ok {
+		return true
 	}
 	for _, cType := range cTypes {
-		if equal := prevLastModified[cType].Equal(curLastModified[cType]); !equal {
+		if content, ok := curVinfo.CveContents[cType]; ok {
+			curLastModified[cType] = content.LastModified
+		}
+	}
+
+	for _, t := range cTypes {
+		if !curLastModified[t].Equal(prevLastModified[t]) {
+			util.Log.Debugf("%s LastModified not equal: \n%s\n%s",
+				cveID, curLastModified[t], prevLastModified[t])
 			return true
 		}
 	}

report/util_test.go

@@ -1,14 +1,23 @@
 package report
 
 import (
+	"os"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
 	"github.com/k0kubun/pp"
 )
 
+func TestMain(m *testing.M) {
+	util.Log = util.NewCustomLogger(config.ServerInfo{})
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestIsCveInfoUpdated(t *testing.T) {
 	f := "2006-01-02"
 	old, _ := time.Parse(f, "2015-12-15")
@@ -33,7 +42,7 @@ func TestIsCveInfoUpdated(t *testing.T) {
 							CveID: "CVE-2017-0001",
 							CveContents: models.NewCveContents(
 								models.CveContent{
-									Type:         models.NVD,
+									Type:         models.NvdXML,
 									CveID:        "CVE-2017-0001",
 									LastModified: time.Time{},
 								},
@@ -47,7 +56,7 @@ func TestIsCveInfoUpdated(t *testing.T) {
 							CveID: "CVE-2017-0001",
 							CveContents: models.NewCveContents(
 								models.CveContent{
-									Type:         models.NVD,
+									Type:         models.NvdXML,
 									CveID:        "CVE-2017-0001",
 									LastModified: time.Time{},
 								},
@@ -68,7 +77,7 @@ func TestIsCveInfoUpdated(t *testing.T) {
 							CveID: "CVE-2017-0002",
 							CveContents: models.NewCveContents(
 								models.CveContent{
-									Type:         models.NVD,
+									Type:         models.Jvn,
 									CveID:        "CVE-2017-0002",
 									LastModified: old,
 								},
@@ -82,7 +91,7 @@ func TestIsCveInfoUpdated(t *testing.T) {
 							CveID: "CVE-2017-0002",
 							CveContents: models.NewCveContents(
 								models.CveContent{
-									Type:         models.NVD,
+									Type:         models.Jvn,
 									CveID:        "CVE-2017-0002",
 									LastModified: old,
 								},
@@ -104,7 +113,7 @@ func TestIsCveInfoUpdated(t *testing.T) {
 							CveID: "CVE-2017-0003",
 							CveContents: models.NewCveContents(
 								models.CveContent{
-									Type:         models.NVD,
+									Type:         models.NvdXML,
 									CveID:        "CVE-2017-0002",
 									LastModified: new,
 								},
@@ -119,7 +128,7 @@ func TestIsCveInfoUpdated(t *testing.T) {
 							CveID: "CVE-2017-0003",
 							CveContents: models.NewCveContents(
 								models.CveContent{
-									Type:         models.NVD,
+									Type:         models.NvdXML,
 									CveID:        "CVE-2017-0002",
 									LastModified: old,
 								},
@@ -141,7 +150,7 @@ func TestIsCveInfoUpdated(t *testing.T) {
 							CveID: "CVE-2017-0004",
 							CveContents: models.NewCveContents(
 								models.CveContent{
-									Type:         models.NVD,
+									Type:         models.NvdXML,
 									CveID:        "CVE-2017-0002",
 									LastModified: old,
 								},
@@ -185,18 +194,18 @@ func TestDiff(t *testing.T) {
 							CveID:            "CVE-2012-6702",
 							AffectedPackages: models.PackageStatuses{{Name: "libexpat1"}},
 							DistroAdvisories: []models.DistroAdvisory{},
-							CpeNames:         []string{},
+							CpeURIs:          []string{},
 						},
 						"CVE-2014-9761": {
 							CveID:            "CVE-2014-9761",
 							AffectedPackages: models.PackageStatuses{{Name: "libc-bin"}},
 							DistroAdvisories: []models.DistroAdvisory{},
-							CpeNames:         []string{},
+							CpeURIs:          []string{},
 						},
 					},
 					Packages: models.Packages{},
 					Errors:   []string{},
-					Optional: [][]interface{}{},
+					Optional: map[string]interface{}{},
 				},
 			},
 			inPrevious: models.ScanResults{
@@ -210,18 +219,18 @@ func TestDiff(t *testing.T) {
 							CveID:            "CVE-2012-6702",
 							AffectedPackages: models.PackageStatuses{{Name: "libexpat1"}},
 							DistroAdvisories: []models.DistroAdvisory{},
-							CpeNames:         []string{},
+							CpeURIs:          []string{},
 						},
 						"CVE-2014-9761": {
 							CveID:            "CVE-2014-9761",
 							AffectedPackages: models.PackageStatuses{{Name: "libc-bin"}},
 							DistroAdvisories: []models.DistroAdvisory{},
-							CpeNames:         []string{},
+							CpeURIs:          []string{},
 						},
 					},
 					Packages: models.Packages{},
 					Errors:   []string{},
-					Optional: [][]interface{}{},
+					Optional: map[string]interface{}{},
 				},
 			},
 			out: models.ScanResult{
@@ -232,7 +241,7 @@ func TestDiff(t *testing.T) {
 				Packages:    models.Packages{},
 				ScannedCves: models.VulnInfos{},
 				Errors:      []string{},
-				Optional:    [][]interface{}{},
+				Optional:    map[string]interface{}{},
 			},
 		},
 		{
@@ -247,7 +256,7 @@ func TestDiff(t *testing.T) {
 							CveID:            "CVE-2016-6662",
 							AffectedPackages: models.PackageStatuses{{Name: "mysql-libs"}},
 							DistroAdvisories: []models.DistroAdvisory{},
-							CpeNames:         []string{},
+							CpeURIs:          []string{},
 						},
 					},
 					Packages: models.Packages{
@@ -285,7 +294,7 @@ func TestDiff(t *testing.T) {
 						CveID:            "CVE-2016-6662",
 						AffectedPackages: models.PackageStatuses{{Name: "mysql-libs"}},
 						DistroAdvisories: []models.DistroAdvisory{},
-						CpeNames:         []string{},
+						CpeURIs:          []string{},
 					},
 				},
 				Packages: models.Packages{
@@ -325,3 +334,104 @@ func TestDiff(t *testing.T) {
 		}
 	}
 }
+
+func TestIsCveFixed(t *testing.T) {
+	type In struct {
+		v    models.VulnInfo
+		prev models.ScanResult
+	}
+	var tests = []struct {
+		in       In
+		expected bool
+	}{
+		{
+			in: In{
+				v: models.VulnInfo{
+					CveID: "CVE-2016-6662",
+					AffectedPackages: models.PackageStatuses{
+						{
+							Name:        "mysql-libs",
+							NotFixedYet: false,
+						},
+					},
+					CveContents: models.NewCveContents(
+						models.CveContent{
+							Type:         models.NvdXML,
+							CveID:        "CVE-2016-6662",
+							LastModified: time.Time{},
+						},
+					),
+				},
+				prev: models.ScanResult{
+					ScannedCves: models.VulnInfos{
+						"CVE-2016-6662": {
+							CveID: "CVE-2016-6662",
+							AffectedPackages: models.PackageStatuses{
+								{
+									Name:        "mysql-libs",
+									NotFixedYet: true,
+								},
+							},
+							CveContents: models.NewCveContents(
+								models.CveContent{
+									Type:         models.NvdXML,
+									CveID:        "CVE-2016-6662",
+									LastModified: time.Time{},
+								},
+							),
+						},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			in: In{
+				v: models.VulnInfo{
+					CveID: "CVE-2016-6662",
+					AffectedPackages: models.PackageStatuses{
+						{
+							Name:        "mysql-libs",
+							NotFixedYet: true,
+						},
+					},
+					CveContents: models.NewCveContents(
+						models.CveContent{
+							Type:         models.NvdXML,
+							CveID:        "CVE-2016-6662",
+							LastModified: time.Time{},
+						},
+					),
+				},
+				prev: models.ScanResult{
+					ScannedCves: models.VulnInfos{
+						"CVE-2016-6662": {
+							CveID: "CVE-2016-6662",
+							AffectedPackages: models.PackageStatuses{
+								{
+									Name:        "mysql-libs",
+									NotFixedYet: true,
+								},
+							},
+							CveContents: models.NewCveContents(
+								models.CveContent{
+									Type:         models.NvdXML,
+									CveID:        "CVE-2016-6662",
+									LastModified: time.Time{},
+								},
+							),
+						},
+					},
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for i, tt := range tests {
+		actual := isCveFixed(tt.in.v, tt.in.prev)
+		if actual != tt.expected {
+			t.Errorf("[%d] actual: %t, expected: %t", i, actual, tt.expected)
+		}
+	}
+}

report/writer.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

scan/alpine.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -64,7 +64,14 @@ func detectAlpine(c config.ServerInfo) (itsMe bool, os osTypeInterface) {
 	return false, os
 }
 
-func (o *alpine) checkDependencies() error {
+func (o *alpine) checkScanMode() error {
+	if o.getServerInfo().Mode.IsOffline() {
+		return fmt.Errorf("Remove offline scan mode, Alpine needs internet connection")
+	}
+	return nil
+}
+
+func (o *alpine) checkDeps() error {
 	o.log.Infof("Dependencies... No need")
 	return nil
 }
@@ -82,6 +89,24 @@ func (o *alpine) apkUpdate() error {
 	return nil
 }
 
+func (o *alpine) preCure() error {
+	o.log.Infof("Scanning in %s", o.getServerInfo().Mode)
+	if err := o.detectIPAddr(); err != nil {
+		o.log.Debugf("Failed to detect IP addresses: %s", err)
+	}
+	// Ignore this error as it just failed to detect the IP addresses
+	return nil
+}
+
+func (o *alpine) postScan() error {
+	return nil
+}
+
+func (o *alpine) detectIPAddr() (err error) {
+	o.ServerInfo.IPv4Addrs, o.ServerInfo.IPv6Addrs, err = o.ip()
+	return err
+}
+
 func (o *alpine) scanPackages() error {
 	if err := o.apkUpdate(); err != nil {
 		return err
@@ -123,6 +148,11 @@ func (o *alpine) scanInstalledPackages() (models.Packages, error) {
 	return o.parseApkInfo(r.Stdout)
 }
 
+func (o *alpine) parseInstalledPackages(stdout string) (models.Packages, models.SrcPackages, error) {
+	installedPackages, err := o.parseApkInfo(stdout)
+	return installedPackages, nil, err
+}
+
 func (o *alpine) parseApkInfo(stdout string) (models.Packages, error) {
 	packs := models.Packages{}
 	scanner := bufio.NewScanner(strings.NewReader(stdout))

scan/amazon.go

@@ -0,0 +1,114 @@
+package scan
+
+import (
+	"fmt"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+)
+
+// inherit OsTypeInterface
+type amazon struct {
+	redhatBase
+}
+
+// NewAmazon is constructor
+func newAmazon(c config.ServerInfo) *amazon {
+	r := &amazon{
+		redhatBase{
+			base: base{
+				osPackages: osPackages{
+					Packages:  models.Packages{},
+					VulnInfos: models.VulnInfos{},
+				},
+			},
+			sudo: rootPrivAmazon{},
+		},
+	}
+	r.log = util.NewCustomLogger(c)
+	r.setServerInfo(c)
+	return r
+}
+
+func (o *amazon) checkScanMode() error {
+	if o.getServerInfo().Mode.IsOffline() {
+		return fmt.Errorf("Remove offline scan mode, Amazon needs internet connection")
+	}
+	return nil
+}
+
+func (o *amazon) checkDeps() error {
+	if o.getServerInfo().Mode.IsFast() {
+		return o.execCheckDeps(o.depsFast())
+	} else if o.getServerInfo().Mode.IsFastRoot() {
+		return o.execCheckDeps(o.depsFastRoot())
+	} else if o.getServerInfo().Mode.IsDeep() {
+		return o.execCheckDeps(o.depsDeep())
+	}
+	return fmt.Errorf("Unknown scan mode")
+}
+
+func (o *amazon) depsFast() []string {
+	if o.getServerInfo().Mode.IsOffline() {
+		return []string{}
+	}
+	// repoquery
+	return []string{"yum-utils"}
+}
+
+func (o *amazon) depsFastRoot() []string {
+	return []string{
+		"yum-utils",
+		"yum-plugin-ps",
+	}
+}
+
+func (o *amazon) depsDeep() []string {
+	return o.depsFastRoot()
+}
+
+func (o *amazon) checkIfSudoNoPasswd() error {
+	if o.getServerInfo().Mode.IsFast() {
+		return o.execCheckIfSudoNoPasswd(o.sudoNoPasswdCmdsFast())
+	} else if o.getServerInfo().Mode.IsFastRoot() {
+		return o.execCheckIfSudoNoPasswd(o.sudoNoPasswdCmdsFastRoot())
+	} else {
+		return o.execCheckIfSudoNoPasswd(o.sudoNoPasswdCmdsDeep())
+	}
+}
+
+func (o *amazon) sudoNoPasswdCmdsFast() []cmd {
+	return []cmd{}
+}
+
+func (o *amazon) sudoNoPasswdCmdsFastRoot() []cmd {
+	return []cmd{
+		{"yum -q ps all --color=never", exitStatusZero},
+		{"stat /proc/1/exe", exitStatusZero},
+		{"needs-restarting", exitStatusZero},
+		{"which which", exitStatusZero},
+	}
+}
+
+func (o *amazon) sudoNoPasswdCmdsDeep() []cmd {
+	return o.sudoNoPasswdCmdsFastRoot()
+}
+
+type rootPrivAmazon struct{}
+
+func (o rootPrivAmazon) repoquery() bool {
+	return false
+}
+
+func (o rootPrivAmazon) yumRepolist() bool {
+	return false
+}
+
+func (o rootPrivAmazon) yumUpdateInfo() bool {
+	return false
+}
+
+func (o rootPrivAmazon) yumChangelog() bool {
+	return false
+}

scan/base.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,7 +18,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package scan
 
 import (
+	"bufio"
 	"fmt"
+	"net"
 	"regexp"
 	"strings"
 	"time"
@@ -32,7 +34,6 @@ type base struct {
 	ServerInfo config.ServerInfo
 	Distro     config.Distro
 	Platform   models.Platform
-
 	osPackages
 
 	log  *logrus.Entry
@@ -97,7 +98,7 @@ func (l *base) runningKernel() (release, version string, err error) {
 }
 
 func (l *base) allContainers() (containers []config.Container, err error) {
-	switch l.ServerInfo.Containers.Type {
+	switch l.ServerInfo.ContainerType {
 	case "", "docker":
 		stdout, err := l.dockerPs("-a --format '{{.ID}} {{.Names}} {{.Image}}'")
 		if err != nil {
@@ -110,14 +111,20 @@ func (l *base) allContainers() (containers []config.Container, err error) {
 			return containers, err
 		}
 		return l.parseLxdPs(stdout)
+	case "lxc":
+		stdout, err := l.lxcPs("-1")
+		if err != nil {
+			return containers, err
+		}
+		return l.parseLxcPs(stdout)
 	default:
 		return containers, fmt.Errorf(
-			"Not supported yet: %s", l.ServerInfo.Containers.Type)
+			"Not supported yet: %s", l.ServerInfo.ContainerType)
 	}
 }
 
 func (l *base) runningContainers() (containers []config.Container, err error) {
-	switch l.ServerInfo.Containers.Type {
+	switch l.ServerInfo.ContainerType {
 	case "", "docker":
 		stdout, err := l.dockerPs("--format '{{.ID}} {{.Names}} {{.Image}}'")
 		if err != nil {
@@ -130,14 +137,20 @@ func (l *base) runningContainers() (containers []config.Container, err error) {
 			return containers, err
 		}
 		return l.parseLxdPs(stdout)
+	case "lxc":
+		stdout, err := l.lxcPs("-1 --running")
+		if err != nil {
+			return containers, err
+		}
+		return l.parseLxcPs(stdout)
 	default:
 		return containers, fmt.Errorf(
-			"Not supported yet: %s", l.ServerInfo.Containers.Type)
+			"Not supported yet: %s", l.ServerInfo.ContainerType)
 	}
 }
 
 func (l *base) exitedContainers() (containers []config.Container, err error) {
-	switch l.ServerInfo.Containers.Type {
+	switch l.ServerInfo.ContainerType {
 	case "", "docker":
 		stdout, err := l.dockerPs("--filter 'status=exited' --format '{{.ID}} {{.Names}} {{.Image}}'")
 		if err != nil {
@@ -150,9 +163,15 @@ func (l *base) exitedContainers() (containers []config.Container, err error) {
 			return containers, err
 		}
 		return l.parseLxdPs(stdout)
+	case "lxc":
+		stdout, err := l.lxcPs("-1 --stopped")
+		if err != nil {
+			return containers, err
+		}
+		return l.parseLxcPs(stdout)
 	default:
 		return containers, fmt.Errorf(
-			"Not supported yet: %s", l.ServerInfo.Containers.Type)
+			"Not supported yet: %s", l.ServerInfo.ContainerType)
 	}
 }
 
@@ -174,6 +193,15 @@ func (l *base) lxdPs(option string) (string, error) {
 	return r.Stdout, nil
 }
 
+func (l *base) lxcPs(option string) (string, error) {
+	cmd := fmt.Sprintf("lxc-ls %s 2>/dev/null", option)
+	r := l.exec(cmd, sudo)
+	if !r.isSuccess() {
+		return "", fmt.Errorf("failed to SSH: %s", r)
+	}
+	return r.Stdout, nil
+}
+
 func (l *base) parseDockerPs(stdout string) (containers []config.Container, err error) {
 	lines := strings.Split(stdout, "\n")
 	for _, line := range lines {
@@ -214,7 +242,64 @@ func (l *base) parseLxdPs(stdout string) (containers []config.Container, err err
 	return
 }
 
+func (l *base) parseLxcPs(stdout string) (containers []config.Container, err error) {
+	lines := strings.Split(stdout, "\n")
+	for _, line := range lines {
+		fields := strings.Fields(line)
+		if len(fields) == 0 {
+			break
+		}
+		containers = append(containers, config.Container{
+			ContainerID: fields[0],
+			Name:        fields[0],
+		})
+	}
+	return
+}
+
+// ip executes ip command and returns IP addresses
+func (l *base) ip() ([]string, []string, error) {
+	// e.g.
+	// 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000\    link/ether 52:54:00:2a:86:4c brd ff:ff:ff:ff:ff:ff
+	// 2: eth0    inet 10.0.2.15/24 brd 10.0.2.255 scope global eth0
+	// 2: eth0    inet6 fe80::5054:ff:fe2a:864c/64 scope link \       valid_lft forever preferred_lft forever
+	r := l.exec("/sbin/ip -o addr", noSudo)
+	if !r.isSuccess() {
+		return nil, nil, fmt.Errorf("Failed to detect IP address: %v", r)
+	}
+	ipv4Addrs, ipv6Addrs := l.parseIP(r.Stdout)
+	return ipv4Addrs, ipv6Addrs, nil
+}
+
+// parseIP parses the results of ip command
+func (l *base) parseIP(stdout string) (ipv4Addrs []string, ipv6Addrs []string) {
+	lines := strings.Split(stdout, "\n")
+	for _, line := range lines {
+		fields := strings.Fields(line)
+		if len(fields) < 4 {
+			continue
+		}
+		ip, _, err := net.ParseCIDR(fields[3])
+		if err != nil {
+			continue
+		}
+		if !ip.IsGlobalUnicast() {
+			continue
+		}
+		if ipv4 := ip.To4(); ipv4 != nil {
+			ipv4Addrs = append(ipv4Addrs, ipv4.String())
+		} else {
+			ipv6Addrs = append(ipv6Addrs, ip.String())
+		}
+	}
+	return
+}
+
 func (l *base) detectPlatform() {
+	if l.getServerInfo().Mode.IsOffline() {
+		l.setPlatform(models.Platform{Name: "unknown"})
+		return
+	}
 	ok, instanceID, err := l.detectRunningOnAws()
 	if err != nil {
 		l.setPlatform(models.Platform{Name: "other"})
@@ -235,7 +320,7 @@ func (l *base) detectPlatform() {
 
 func (l *base) detectRunningOnAws() (ok bool, instanceID string, err error) {
 	if r := l.exec("type curl", noSudo); r.isSuccess() {
-		cmd := "curl --max-time 1 --retry 3 --noproxy 169.254.169.254 http://169.254.169.254/latest/meta-data/instance-id"
+		cmd := "curl --max-time 1 --noproxy 169.254.169.254 http://169.254.169.254/latest/meta-data/instance-id"
 		r := l.exec(cmd, noSudo)
 		if r.isSuccess() {
 			id := strings.TrimSpace(r.Stdout)
@@ -286,7 +371,7 @@ func (l *base) isAwsInstanceID(str string) bool {
 }
 
 func (l *base) convertToModel() models.ScanResult {
-	ctype := l.ServerInfo.Containers.Type
+	ctype := l.ServerInfo.ContainerType
 	if l.ServerInfo.Container.ContainerID != "" && ctype == "" {
 		ctype = "docker"
 	}
@@ -306,10 +391,13 @@ func (l *base) convertToModel() models.ScanResult {
 		JSONVersion:   models.JSONVersion,
 		ServerName:    l.ServerInfo.ServerName,
 		ScannedAt:     time.Now(),
+		ScanMode:      l.ServerInfo.Mode.String(),
 		Family:        l.Distro.Family,
 		Release:       l.Distro.Release,
 		Container:     container,
 		Platform:      l.Platform,
+		IPv4Addrs:     l.ServerInfo.IPv4Addrs,
+		IPv6Addrs:     l.ServerInfo.IPv6Addrs,
 		ScannedCves:   l.VulnInfos,
 		RunningKernel: l.Kernel,
 		Packages:      l.Packages,
@@ -326,3 +414,62 @@ func (l *base) setErrs(errs []error) {
 func (l *base) getErrs() []error {
 	return l.errs
 }
+
+const (
+	systemd  = "systemd"
+	upstart  = "upstart"
+	sysVinit = "init"
+)
+
+// https://unix.stackexchange.com/questions/196166/how-to-find-out-if-a-system-uses-sysv-upstart-or-systemd-initsystem
+func (l *base) detectInitSystem() (string, error) {
+	var f func(string) (string, error)
+	f = func(cmd string) (string, error) {
+		r := l.exec(cmd, sudo)
+		if !r.isSuccess() {
+			return "", fmt.Errorf("Failed to stat %s: %s", cmd, r)
+		}
+		scanner := bufio.NewScanner(strings.NewReader(r.Stdout))
+		scanner.Scan()
+		line := strings.TrimSpace(scanner.Text())
+		if strings.Contains(line, "systemd") {
+			return systemd, nil
+		} else if strings.Contains(line, "upstart") {
+			return upstart, nil
+		} else if strings.Contains(line, "File: ‘/proc/1/exe’ -> ‘/sbin/init’") ||
+			strings.Contains(line, "File: `/proc/1/exe' -> `/sbin/init'") {
+			return f("stat /sbin/init")
+		} else if line == "File: ‘/sbin/init’" ||
+			line == "File: `/sbin/init'" {
+			r := l.exec("/sbin/init --version", noSudo)
+			if r.isSuccess() {
+				if strings.Contains(r.Stdout, "upstart") {
+					return upstart, nil
+				}
+			}
+			return sysVinit, nil
+		}
+		return "", fmt.Errorf("Failed to detect a init system: %s", line)
+	}
+	return f("stat /proc/1/exe")
+}
+
+func (l *base) detectServiceName(pid string) (string, error) {
+	cmd := fmt.Sprintf("systemctl status --quiet --no-pager %s", pid)
+	r := l.exec(cmd, noSudo)
+	if !r.isSuccess() {
+		return "", fmt.Errorf("Failed to stat %s: %s", cmd, r)
+	}
+	return l.parseSystemctlStatus(r.Stdout), nil
+}
+
+func (l *base) parseSystemctlStatus(stdout string) string {
+	scanner := bufio.NewScanner(strings.NewReader(stdout))
+	scanner.Scan()
+	line := scanner.Text()
+	ss := strings.Fields(line)
+	if len(ss) < 2 || strings.HasPrefix(line, "Failed to get unit for PID") {
+		return ""
+	}
+	return ss[1]
+}

scan/base_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -25,7 +25,6 @@ import (
 )
 
 func TestParseDockerPs(t *testing.T) {
-
 	var test = struct {
 		in       string
 		expected []config.Container
@@ -46,7 +45,7 @@ f570ae647edc agitated_lovelace centos:latest`,
 		},
 	}
 
-	r := newRedhat(config.ServerInfo{})
+	r := newRHEL(config.ServerInfo{})
 	actual, err := r.parseDockerPs(test.in)
 	if err != nil {
 		t.Errorf("Error occurred. in: %s, err: %s", test.in, err)
@@ -60,7 +59,6 @@ f570ae647edc agitated_lovelace centos:latest`,
 }
 
 func TestParseLxdPs(t *testing.T) {
-
 	var test = struct {
 		in       string
 		expected []config.Container
@@ -84,7 +82,7 @@ func TestParseLxdPs(t *testing.T) {
 		},
 	}
 
-	r := newRedhat(config.ServerInfo{})
+	r := newRHEL(config.ServerInfo{})
 	actual, err := r.parseLxdPs(test.in)
 	if err != nil {
 		t.Errorf("Error occurred. in: %s, err: %s", test.in, err)
@@ -97,6 +95,36 @@ func TestParseLxdPs(t *testing.T) {
 	}
 }
 
+func TestParseIp(t *testing.T) {
+
+	var test = struct {
+		in        string
+		expected4 []string
+		expected6 []string
+	}{
+		in: `1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN \    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
+1: lo    inet 127.0.0.1/8 scope host lo
+1: lo    inet6 ::1/128 scope host \       valid_lft forever preferred_lft forever
+2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000\    link/ether 52:54:00:2a:86:4c brd ff:ff:ff:ff:ff:ff
+2: eth0    inet 10.0.2.15/24 brd 10.0.2.255 scope global eth0
+2: eth0    inet6 fe80::5054:ff:fe2a:864c/64 scope link \       valid_lft forever preferred_lft forever
+3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000\    link/ether 08:00:27:36:76:60 brd ff:ff:ff:ff:ff:ff
+3: eth1    inet 192.168.33.11/24 brd 192.168.33.255 scope global eth1
+3: eth1    inet6 2001:db8::68/64 scope link \       valid_lft forever preferred_lft forever `,
+		expected4: []string{"10.0.2.15", "192.168.33.11"},
+		expected6: []string{"2001:db8::68"},
+	}
+
+	r := newRHEL(config.ServerInfo{})
+	actual4, actual6 := r.parseIP(test.in)
+	if !reflect.DeepEqual(test.expected4, actual4) {
+		t.Errorf("expected %v, actual %v", test.expected4, actual4)
+	}
+	if !reflect.DeepEqual(test.expected6, actual6) {
+		t.Errorf("expected %v, actual %v", test.expected6, actual6)
+	}
+}
+
 func TestIsAwsInstanceID(t *testing.T) {
 	var tests = []struct {
 		in       string
@@ -110,7 +138,7 @@ func TestIsAwsInstanceID(t *testing.T) {
 		{"no data", false},
 	}
 
-	r := newRedhat(config.ServerInfo{})
+	r := newAmazon(config.ServerInfo{})
 	for _, tt := range tests {
 		actual := r.isAwsInstanceID(tt.in)
 		if tt.expected != actual {
@@ -118,3 +146,35 @@ func TestIsAwsInstanceID(t *testing.T) {
 		}
 	}
 }
+
+func TestParseSystemctlStatus(t *testing.T) {
+	var tests = []struct {
+		in  string
+		out string
+	}{
+		{
+			in: `● NetworkManager.service - Network Manager
+   Loaded: loaded (/usr/lib/systemd/system/NetworkManager.service; enabled; vendor preset: enabled)
+   Active: active (running) since Wed 2018-01-10 17:15:39 JST; 2 months 10 days ago
+     Docs: man:NetworkManager(8)
+ Main PID: 437 (NetworkManager)
+   Memory: 424.0K
+   CGroup: /system.slice/NetworkManager.service
+           ├─437 /usr/sbin/NetworkManager --no-daemon
+           └─572 /sbin/dhclient -d -q -sf /usr/libexec/nm-dhcp-helper -pf /var/run/dhclient-ens160.pid -lf /var/lib/NetworkManager/dhclient-241ed966-e1c7-4d5c-a6a0-8a6dba457277-ens160.lease -cf /var/lib/NetworkManager/dhclient-ens160.conf ens160`,
+			out: "NetworkManager.service",
+		},
+		{
+			in:  `Failed to get unit for PID 700: PID 700 does not belong to any loaded unit.`,
+			out: "",
+		},
+	}
+
+	r := newCentOS(config.ServerInfo{})
+	for _, tt := range tests {
+		actual := r.parseSystemctlStatus(tt.in)
+		if tt.out != actual {
+			t.Errorf("expected %v, actual %v", tt.out, actual)
+		}
+	}
+}

scan/centos.go

@@ -0,0 +1,120 @@
+package scan
+
+import (
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+)
+
+// inherit OsTypeInterface
+type centos struct {
+	redhatBase
+}
+
+// NewAmazon is constructor
+func newCentOS(c config.ServerInfo) *centos {
+	r := &centos{
+		redhatBase{
+			base: base{
+				osPackages: osPackages{
+					Packages:  models.Packages{},
+					VulnInfos: models.VulnInfos{},
+				},
+			},
+			sudo: rootPrivCentos{},
+		},
+	}
+	r.log = util.NewCustomLogger(c)
+	r.setServerInfo(c)
+	return r
+}
+
+func (o *centos) checkScanMode() error {
+	return nil
+}
+
+func (o *centos) checkDeps() error {
+	if o.getServerInfo().Mode.IsFast() {
+		return o.execCheckDeps(o.depsFast())
+	} else if o.getServerInfo().Mode.IsFastRoot() {
+		return o.execCheckDeps(o.depsFastRoot())
+	} else {
+		return o.execCheckDeps(o.depsDeep())
+	}
+}
+
+func (o *centos) depsFast() []string {
+	if o.getServerInfo().Mode.IsOffline() {
+		return []string{}
+	}
+	// repoquery
+	return []string{"yum-utils"}
+}
+
+func (o *centos) depsFastRoot() []string {
+	return []string{
+		"yum-utils",
+		"yum-plugin-ps",
+	}
+}
+
+func (o *centos) depsDeep() []string {
+	return []string{
+		"yum-utils",
+		"yum-plugin-ps",
+		"yum-plugin-changelog",
+	}
+}
+
+func (o *centos) checkIfSudoNoPasswd() error {
+	if o.getServerInfo().Mode.IsFast() {
+		return o.execCheckIfSudoNoPasswd(o.sudoNoPasswdCmdsFast())
+	} else if o.getServerInfo().Mode.IsFastRoot() {
+		return o.execCheckIfSudoNoPasswd(o.sudoNoPasswdCmdsFastRoot())
+	} else {
+		return o.execCheckIfSudoNoPasswd(o.sudoNoPasswdCmdsDeep())
+	}
+}
+
+func (o *centos) sudoNoPasswdCmdsFast() []cmd {
+	return []cmd{}
+}
+
+func (o *centos) sudoNoPasswdCmdsFastRoot() []cmd {
+	if o.getServerInfo().Mode.IsOffline() {
+		// yum ps needs internet connection
+		return []cmd{
+			{"stat /proc/1/exe", exitStatusZero},
+			{"needs-restarting", exitStatusZero},
+			{"which which", exitStatusZero},
+		}
+	}
+	return []cmd{
+		{"yum -q ps all --color=never", exitStatusZero},
+		{"stat /proc/1/exe", exitStatusZero},
+		{"needs-restarting", exitStatusZero},
+		{"which which", exitStatusZero},
+	}
+}
+
+func (o *centos) sudoNoPasswdCmdsDeep() []cmd {
+	return o.sudoNoPasswdCmdsFastRoot()
+}
+
+type rootPrivCentos struct{}
+
+func (o rootPrivCentos) repoquery() bool {
+	return false
+}
+
+func (o rootPrivCentos) yumRepolist() bool {
+	return false
+}
+
+func (o rootPrivCentos) yumUpdateInfo() bool {
+	return false
+}
+
+func (o rootPrivCentos) yumChangelog() bool {
+	return false
+}

scan/debian.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -62,7 +62,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
 			return false, deb, nil
 		}
 		if r.ExitStatus == 255 {
-			return false, deb, fmt.Errorf("Unable to connect via SSH. Check SSH settings. If you have never SSH to the host to be scanned, SSH to the host before scanning in order to add the HostKey. %s@%s port: %s\n%s", c.User, c.Host, c.Port, r)
+			return false, deb, fmt.Errorf("Unable to connect via SSH. Scan with -vvv option to print SSH debugging messages and check SSH settings. If you have never SSH to the host to be scanned, SSH to the host before scanning in order to add the HostKey. %s@%s port: %s\n%s", c.User, c.Host, c.Port, r)
 		}
 		util.Log.Debugf("Not Debian like Linux. %s", r)
 		return false, deb, nil
@@ -135,57 +135,143 @@ func trim(str string) string {
 	return strings.TrimSpace(str)
 }
 
+func (o *debian) checkScanMode() error {
+	return nil
+}
+
 func (o *debian) checkIfSudoNoPasswd() error {
-	if config.Conf.Deep || o.Distro.Family == config.Raspbian {
-		cmd := util.PrependProxyEnv("apt-get update")
+	if o.getServerInfo().Mode.IsFast() {
+		o.log.Infof("sudo ... No need")
+		return nil
+	}
+
+	cmds := []string{
+		"checkrestart",
+		"stat /proc/1/exe",
+	}
+
+	if !o.getServerInfo().Mode.IsOffline() {
+		cmds = append(cmds, "apt-get update")
+	}
+
+	for _, cmd := range cmds {
+		cmd = util.PrependProxyEnv(cmd)
 		o.log.Infof("Checking... sudo %s", cmd)
 		r := o.exec(cmd, sudo)
 		if !r.isSuccess() {
 			o.log.Errorf("sudo error on %s", r)
 			return fmt.Errorf("Failed to sudo: %s", r)
 		}
-		o.log.Infof("Sudo... Pass")
-		return nil
 	}
 
-	o.log.Infof("sudo ... No need")
+	initName, err := o.detectInitSystem()
+	if initName == upstart && err == nil {
+		cmd := util.PrependProxyEnv("initctl status --help")
+		o.log.Infof("Checking... sudo %s", cmd)
+		r := o.exec(cmd, sudo)
+		if !r.isSuccess() {
+			o.log.Errorf("sudo error on %s", r)
+			return fmt.Errorf("Failed to sudo: %s", r)
+		}
+	}
+
+	o.log.Infof("Sudo... Pass")
 	return nil
 }
 
-func (o *debian) checkDependencies() error {
-	packNames := []string{}
+type dep struct {
+	packName      string
+	required      bool
+	logFunc       func(string, ...interface{})
+	additionalMsg string
+}
 
-	switch o.Distro.Family {
-	case config.Ubuntu, config.Raspbian:
-		o.log.Infof("Dependencies... No need")
-		return nil
-
-	case config.Debian:
-		// https://askubuntu.com/a/742844
-		packNames = append(packNames, "reboot-notifier")
-
-		if !config.Conf.Deep {
-			// Debian needs aptitude to get changelogs.
-			// Because unable to get changelogs via apt-get changelog on Debian.
-			packNames = append(packNames, "aptitude")
-		}
-
-	default:
-		return fmt.Errorf("Not implemented yet: %s", o.Distro)
+func (o *debian) checkDeps() error {
+	deps := []dep{}
+	if o.getServerInfo().Mode.IsDeep() || o.getServerInfo().Mode.IsFastRoot() {
+		// checkrestart
+		deps = append(deps, dep{
+			packName: "debian-goodies",
+			required: true,
+			logFunc:  o.log.Errorf,
+		})
 	}
 
-	for _, name := range packNames {
-		cmd := "dpkg-query -W " + name
-		if r := o.exec(cmd, noSudo); !r.isSuccess() {
-			msg := fmt.Sprintf("%s is not installed", name)
-			o.log.Errorf(msg)
-			return fmt.Errorf(msg)
+	if o.Distro.Family == config.Debian {
+		// https://askubuntu.com/a/742844
+		if !o.ServerInfo.IsContainer() {
+			deps = append(deps, dep{
+				packName:      "reboot-notifier",
+				required:      false,
+				logFunc:       o.log.Warnf,
+				additionalMsg: ". Install it if you want to detect whether not rebooted after kernel update. To install `reboot-notifier` on Debian, see https://feeding.cloud.geek.nz/posts/introducing-reboot-notifier/",
+			})
 		}
+
+		// Changelogs will be fetched only in deep scan mode
+		if o.getServerInfo().Mode.IsDeep() {
+			// Debian needs aptitude to get changelogs.
+			// Because unable to get changelogs via `apt-get changelog` on Debian.
+			deps = append(deps, dep{
+				packName: "aptitude",
+				required: true,
+				logFunc:  o.log.Errorf,
+			})
+		}
+	}
+
+	for _, dep := range deps {
+		cmd := fmt.Sprintf("%s %s", dpkgQuery, dep.packName)
+		msg := fmt.Sprintf("%s is not installed", dep.packName)
+		r := o.exec(cmd, noSudo)
+		if !r.isSuccess() {
+			if dep.additionalMsg != "" {
+				msg += dep.additionalMsg
+			}
+			dep.logFunc(msg)
+			if dep.required {
+				return fmt.Errorf(msg)
+			}
+			continue
+		}
+
+		_, status, _, _, _, _ := o.parseScannedPackagesLine(r.Stdout)
+		if status != "ii" {
+			if dep.additionalMsg != "" {
+				msg += dep.additionalMsg
+			}
+			dep.logFunc(msg)
+			if dep.required {
+				return fmt.Errorf(msg)
+			}
+		}
+
 	}
 	o.log.Infof("Dependencies... Pass")
 	return nil
 }
 
+func (o *debian) preCure() error {
+	o.log.Infof("Scanning in %s", o.getServerInfo().Mode)
+	if err := o.detectIPAddr(); err != nil {
+		o.log.Debugf("Failed to detect IP addresses: %s", err)
+	}
+	// Ignore this error as it just failed to detect the IP addresses
+	return nil
+}
+
+func (o *debian) postScan() error {
+	if o.getServerInfo().Mode.IsDeep() || o.getServerInfo().Mode.IsFastRoot() {
+		return o.checkrestart()
+	}
+	return nil
+}
+
+func (o *debian) detectIPAddr() (err error) {
+	o.ServerInfo.IPv4Addrs, o.ServerInfo.IPv6Addrs, err = o.ip()
+	return err
+}
+
 func (o *debian) scanPackages() error {
 	// collect the running kernel information
 	release, version, err := o.runningKernel()
@@ -212,7 +298,11 @@ func (o *debian) scanPackages() error {
 	o.Packages = installed
 	o.SrcPackages = srcPacks
 
-	if config.Conf.Deep || o.Distro.Family == config.Raspbian {
+	if o.getServerInfo().Mode.IsOffline() {
+		return nil
+	}
+
+	if o.getServerInfo().Mode.IsDeep() || o.Distro.Family == config.Raspbian {
 		unsecures, err := o.scanUnsecurePackages(updatable)
 		if err != nil {
 			o.log.Errorf("Failed to scan vulnerable packages: %s", err)
@@ -237,23 +327,63 @@ func (o *debian) rebootRequired() (bool, error) {
 	}
 }
 
+const dpkgQuery = `dpkg-query -W -f="\${binary:Package},\${db:Status-Abbrev},\${Version},\${Source},\${source:Version}\n"`
+
 func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, models.SrcPackages, error) {
-	installed, updatable, srcPacks := models.Packages{}, models.Packages{}, models.SrcPackages{}
-	r := o.exec(`dpkg-query -W -f="\${binary:Package},\${db:Status-Abbrev},\${Version},\${Source},\${source:Version}\n"`, noSudo)
+	updatable := models.Packages{}
+	r := o.exec(dpkgQuery, noSudo)
 	if !r.isSuccess() {
 		return nil, nil, nil, fmt.Errorf("Failed to SSH: %s", r)
 	}
 
+	installed, srcPacks, err := o.parseInstalledPackages(r.Stdout)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	if o.getServerInfo().Mode.IsOffline() || o.getServerInfo().Mode.IsFast() {
+		return installed, updatable, srcPacks, nil
+	}
+
+	if err := o.aptGetUpdate(); err != nil {
+		return nil, nil, nil, err
+	}
+	updatableNames, err := o.getUpdatablePackNames()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	for _, name := range updatableNames {
+		for _, pack := range installed {
+			if pack.Name == name {
+				updatable[name] = pack
+				break
+			}
+		}
+	}
+
+	// Fill the candidate versions of upgradable packages
+	err = o.fillCandidateVersion(updatable)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err)
+	}
+	installed.MergeNewVersion(updatable)
+
+	return installed, updatable, srcPacks, nil
+}
+
+func (o *debian) parseInstalledPackages(stdout string) (models.Packages, models.SrcPackages, error) {
+	installed, srcPacks := models.Packages{}, models.SrcPackages{}
+
 	// e.g.
 	// curl,ii ,7.38.0-4+deb8u2,,7.38.0-4+deb8u2
 	// openssh-server,ii ,1:6.7p1-5+deb8u3,openssh,1:6.7p1-5+deb8u3
 	// tar,ii ,1.27.1-2+b1,tar (1.27.1-2),1.27.1-2
-	lines := strings.Split(r.Stdout, "\n")
+	lines := strings.Split(stdout, "\n")
 	for _, line := range lines {
 		if trimmed := strings.TrimSpace(line); len(trimmed) != 0 {
 			name, status, version, srcName, srcVersion, err := o.parseScannedPackagesLine(trimmed)
-			if err != nil {
-				return nil, nil, nil, fmt.Errorf(
+			if err != nil || len(status) < 2 {
+				return nil, nil, fmt.Errorf(
 					"Debian: Failed to parse package line: %s", line)
 			}
 
@@ -299,28 +429,7 @@ func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, mode
 	for name := range installed {
 		delete(srcPacks, name)
 	}
-
-	updatableNames, err := o.getUpdatablePackNames()
-	if err != nil {
-		return nil, nil, nil, err
-	}
-	for _, name := range updatableNames {
-		for _, pack := range installed {
-			if pack.Name == name {
-				updatable[name] = pack
-				break
-			}
-		}
-	}
-
-	// Fill the candidate versions of upgradable packages
-	err = o.fillCandidateVersion(updatable)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err)
-	}
-	installed.MergeNewVersion(updatable)
-
-	return installed, updatable, srcPacks, nil
+	return installed, srcPacks, nil
 }
 
 func (o *debian) parseScannedPackagesLine(line string) (name, status, version, srcName, srcVersion string, err error) {
@@ -331,7 +440,7 @@ func (o *debian) parseScannedPackagesLine(line string) (name, status, version, s
 		if i := strings.IndexRune(name, ':'); i >= 0 {
 			name = name[:i]
 		}
-		status = ss[1]
+		status = strings.TrimSpace(ss[1])
 		version = ss[2]
 		// remove version. ex: tar (1.27.1-2)
 		srcName = strings.Split(ss[3], " ")[0]
@@ -346,14 +455,12 @@ func (o *debian) aptGetUpdate() error {
 	o.log.Infof("apt-get update...")
 	cmd := util.PrependProxyEnv("apt-get update")
 	if r := o.exec(cmd, sudo); !r.isSuccess() {
-		return fmt.Errorf("Failed to SSH: %s", r)
+		return fmt.Errorf("Failed to apt-get update: %s", r)
 	}
 	return nil
 }
 
 func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) {
-	o.aptGetUpdate()
-
 	// Setup changelog cache
 	current := cache.Meta{
 		Name:   o.getServerInfo().GetServerName(),
@@ -368,7 +475,7 @@ func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInf
 	}
 
 	// Collect CVE information of upgradable packages
-	vulnInfos, err := o.scanVulnInfos(updatable, meta)
+	vulnInfos, err := o.scanChangelogs(updatable, meta)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err)
 	}
@@ -498,7 +605,7 @@ type DetectedCveID struct {
 	Confidence models.Confidence
 }
 
-func (o *debian) scanVulnInfos(updatablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) {
+func (o *debian) scanChangelogs(updatablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) {
 	type response struct {
 		pack           *models.Package
 		DetectedCveIDs []DetectedCveID
@@ -534,7 +641,7 @@ func (o *debian) scanVulnInfos(updatablePacks models.Packages, meta *cache.Meta)
 					// if the changelog is not in cache or failed to get from local cache,
 					// get the changelog of the package via internet.
 					// After that, store it in the cache.
-					if cveIDs, pack, err := o.scanPackageCveIDs(p); err != nil {
+					if cveIDs, pack, err := o.fetchParseChangelog(p); err != nil {
 						errChan <- err
 					} else {
 						resChan <- response{pack, cveIDs}
@@ -590,7 +697,7 @@ func (o *debian) scanVulnInfos(updatablePacks models.Packages, meta *cache.Meta)
 
 		vinfos[cveID.CveID] = models.VulnInfo{
 			CveID:            cveID.CveID,
-			Confidence:       cveID.Confidence,
+			Confidences:      models.Confidences{cveID.Confidence},
 			AffectedPackages: affected,
 		}
 	}
@@ -632,7 +739,7 @@ func (o *debian) getChangelogCache(meta *cache.Meta, pack models.Package) string
 	return changelog
 }
 
-func (o *debian) scanPackageCveIDs(pack models.Package) ([]DetectedCveID, *models.Package, error) {
+func (o *debian) fetchParseChangelog(pack models.Package) ([]DetectedCveID, *models.Package, error) {
 	cmd := ""
 	switch o.Distro.Family {
 	case config.Ubuntu, config.Raspbian:
@@ -711,7 +818,7 @@ func (o *debian) getCveIDsFromChangelog(
 
 	// Only logging the error.
 	o.log.Warnf("Failed to find the version in changelog: %s-%s", name, ver)
-	o.log.Debugf("Changelog of : %s-%s", name, ver, changelog)
+	o.log.Debugf("Changelog of %s-%s: %s", name, ver, changelog)
 
 	// If the version is not in changelog, return entire changelog to put into cache
 	pack := o.Packages[name]
@@ -837,3 +944,145 @@ func (o *debian) parseAptCachePolicy(stdout, name string) (packCandidateVer, err
 	}
 	return ver, fmt.Errorf("Unknown Format: %s", stdout)
 }
+
+func (o *debian) checkrestart() error {
+	initName, err := o.detectInitSystem()
+	if err != nil {
+		o.log.Warn(err)
+		// continue scanning
+	}
+
+	cmd := "LANGUAGE=en_US.UTF-8 checkrestart"
+	r := o.exec(cmd, sudo)
+	if !r.isSuccess() {
+		return fmt.Errorf(
+			"Failed to %s. status: %d, stdout: %s, stderr: %s",
+			cmd, r.ExitStatus, r.Stdout, r.Stderr)
+	}
+	packs, unknownServices := o.parseCheckRestart(r.Stdout)
+	pidService := map[string]string{}
+	if initName == upstart {
+		for _, s := range unknownServices {
+			cmd := "LANGUAGE=en_US.UTF-8 initctl status " + s
+			r := o.exec(cmd, sudo)
+			if !r.isSuccess() {
+				continue
+			}
+			if ss := strings.Fields(r.Stdout); len(ss) == 4 && ss[2] == "process" {
+				pidService[ss[3]] = s
+			}
+		}
+	}
+
+	for i, p := range packs {
+		pack := o.Packages[p.Name]
+		pack.NeedRestartProcs = p.NeedRestartProcs
+		o.Packages[p.Name] = pack
+
+		for j, proc := range p.NeedRestartProcs {
+			if proc.HasInit == false {
+				continue
+			}
+			packs[i].NeedRestartProcs[j].InitSystem = initName
+			if initName == systemd {
+				name, err := o.detectServiceName(proc.PID)
+				if err != nil {
+					o.log.Warn(err)
+					// continue scanning
+				}
+				packs[i].NeedRestartProcs[j].ServiceName = name
+			} else {
+				if proc.ServiceName == "" {
+					if ss := strings.Fields(r.Stdout); len(ss) == 4 && ss[2] == "process" {
+						if name, ok := pidService[ss[3]]; ok {
+							packs[i].NeedRestartProcs[j].ServiceName = name
+						}
+					}
+				}
+			}
+		}
+		o.Packages[p.Name] = p
+	}
+	return nil
+}
+
+func (o *debian) parseCheckRestart(stdout string) (models.Packages, []string) {
+	services := []string{}
+	scanner := bufio.NewScanner(strings.NewReader(stdout))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "service") && strings.HasSuffix(line, "restart") {
+			ss := strings.Fields(line)
+			if len(ss) != 3 {
+				continue
+			}
+			services = append(services, ss[1])
+		}
+	}
+
+	packs := models.Packages{}
+	packName := ""
+	hasInit := true
+	scanner = bufio.NewScanner(strings.NewReader(stdout))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasSuffix(line, "do not seem to have an associated init script to restart them:") {
+			hasInit = false
+			continue
+		}
+		if strings.HasSuffix(line, ":") && len(strings.Fields(line)) == 1 {
+			packName = strings.TrimSuffix(line, ":")
+			continue
+		}
+		if strings.HasPrefix(line, "\t") {
+			ss := strings.Fields(line)
+			if len(ss) != 2 {
+				continue
+			}
+
+			serviceName := ""
+			for _, s := range services {
+				if packName == s {
+					serviceName = s
+				}
+			}
+			if p, ok := packs[packName]; ok {
+				p.NeedRestartProcs = append(p.NeedRestartProcs, models.NeedRestartProcess{
+					PID:         ss[0],
+					Path:        ss[1],
+					ServiceName: serviceName,
+					HasInit:     hasInit,
+				})
+				packs[packName] = p
+			} else {
+				packs[packName] = models.Package{
+					Name: packName,
+					NeedRestartProcs: []models.NeedRestartProcess{
+						{
+							PID:         ss[0],
+							Path:        ss[1],
+							ServiceName: serviceName,
+							HasInit:     hasInit,
+						},
+					},
+				}
+			}
+		}
+	}
+
+	unknownServices := []string{}
+	for _, s := range services {
+		found := false
+		for _, p := range packs {
+			for _, proc := range p.NeedRestartProcs {
+				if proc.ServiceName == s {
+					found = true
+				}
+			}
+		}
+		if !found {
+			unknownServices = append(unknownServices, s)
+		}
+	}
+	return packs, unknownServices
+}

scan/debian_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -160,10 +160,10 @@ systemd (228-5) unstable; urgency=medium`,
 		 util-linux (2.27-3ubuntu1) xenial; urgency=medium`,
 			},
 			[]DetectedCveID{
-			// {"CVE-2015-2325", models.ChangelogLenientMatch},
-			// {"CVE-2015-2326", models.ChangelogLenientMatch},
-			// {"CVE-2015-3210", models.ChangelogLenientMatch},
-			// {"CVE-2016-1000000", models.ChangelogLenientMatch},
+				// {"CVE-2015-2325", models.ChangelogLenientMatch},
+				// {"CVE-2015-2326", models.ChangelogLenientMatch},
+				// {"CVE-2015-3210", models.ChangelogLenientMatch},
+				// {"CVE-2016-1000000", models.ChangelogLenientMatch},
 			},
 			models.Changelog{
 				// Contents: `util-linux (2.27.1-3ubuntu1) xenial; urgency=medium
@@ -196,10 +196,10 @@ systemd (228-5) unstable; urgency=medium`,
 		 util-linux (2.27-3) xenial; urgency=medium`,
 			},
 			[]DetectedCveID{
-			// {"CVE-2015-2325", models.ChangelogLenientMatch},
-			// {"CVE-2015-2326", models.ChangelogLenientMatch},
-			// {"CVE-2015-3210", models.ChangelogLenientMatch},
-			// {"CVE-2016-1000000", models.ChangelogLenientMatch},
+				// {"CVE-2015-2325", models.ChangelogLenientMatch},
+				// {"CVE-2015-2326", models.ChangelogLenientMatch},
+				// {"CVE-2015-3210", models.ChangelogLenientMatch},
+				// {"CVE-2016-1000000", models.ChangelogLenientMatch},
 			},
 			models.Changelog{
 				// Contents: `util-linux (2.27.1-3ubuntu1) xenial; urgency=medium
@@ -587,3 +587,139 @@ func TestParseAptCachePolicy(t *testing.T) {
 		}
 	}
 }
+
+func TestParseCheckRestart(t *testing.T) {
+	r := newDebian(config.ServerInfo{})
+	r.Distro = config.Distro{Family: "debian"}
+	var tests = []struct {
+		in              string
+		out             models.Packages
+		unknownServices []string
+	}{
+		{
+			in: `Found 27 processes using old versions of upgraded files
+(19 distinct programs)
+(15 distinct packages)
+
+Of these, 14 seem to contain systemd service definitions or init scripts which can be used to restart them.
+The following packages seem to have definitions that could be used
+to restart their services:
+varnish:
+	3490	/usr/sbin/varnishd
+	3704	/usr/sbin/varnishd
+memcached:
+	3636	/usr/bin/memcached
+openssh-server:
+	1252	/usr/sbin/sshd
+	1184	/usr/sbin/sshd
+accountsservice:
+	462     /usr/lib/accountsservice/accounts-daemon
+
+These are the systemd services:
+systemctl restart accounts-daemon.service
+
+These are the initd scripts:
+service varnish restart
+service memcached restart
+service ssh restart
+
+These processes (1) do not seem to have an associated init script to restart them:
+util-linux:
+	3650	/sbin/agetty
+	3648	/sbin/agetty`,
+			out: models.NewPackages(
+				models.Package{
+					Name: "varnish",
+					NeedRestartProcs: []models.NeedRestartProcess{
+						{
+							PID:         "3490",
+							Path:        "/usr/sbin/varnishd",
+							ServiceName: "varnish",
+							HasInit:     true,
+						},
+						{
+							PID:         "3704",
+							Path:        "/usr/sbin/varnishd",
+							ServiceName: "varnish",
+							HasInit:     true,
+						},
+					},
+				},
+				models.Package{
+					Name: "memcached",
+					NeedRestartProcs: []models.NeedRestartProcess{
+						{
+							PID:         "3636",
+							Path:        "/usr/bin/memcached",
+							ServiceName: "memcached",
+							HasInit:     true,
+						},
+					},
+				},
+				models.Package{
+					Name: "openssh-server",
+					NeedRestartProcs: []models.NeedRestartProcess{
+						{
+							PID:         "1252",
+							Path:        "/usr/sbin/sshd",
+							ServiceName: "",
+							HasInit:     true,
+						},
+						{
+							PID:         "1184",
+							Path:        "/usr/sbin/sshd",
+							ServiceName: "",
+							HasInit:     true,
+						},
+					},
+				},
+				models.Package{
+					Name: "accountsservice",
+					NeedRestartProcs: []models.NeedRestartProcess{
+						{
+							PID:         "462",
+							Path:        "/usr/lib/accountsservice/accounts-daemon",
+							ServiceName: "",
+							HasInit:     true,
+						},
+					},
+				},
+				models.Package{
+					Name: "util-linux",
+					NeedRestartProcs: []models.NeedRestartProcess{
+						{
+							PID:     "3650",
+							Path:    "/sbin/agetty",
+							HasInit: false,
+						},
+						{
+							PID:     "3648",
+							Path:    "/sbin/agetty",
+							HasInit: false,
+						},
+					},
+				},
+			),
+			unknownServices: []string{"ssh"},
+		},
+		{
+			in:              `Found 0 processes using old versions of upgraded files`,
+			out:             models.Packages{},
+			unknownServices: []string{},
+		},
+	}
+
+	for _, tt := range tests {
+		packages, services := r.parseCheckRestart(tt.in)
+		for name, ePack := range tt.out {
+			if !reflect.DeepEqual(ePack, packages[name]) {
+				e := pp.Sprintf("%v", ePack)
+				a := pp.Sprintf("%v", packages[name])
+				t.Errorf("expected %s, actual %s", e, a)
+			}
+		}
+		if !reflect.DeepEqual(tt.unknownServices, services) {
+			t.Errorf("expected %s, actual %s", tt.unknownServices, services)
+		}
+	}
+}

scan/executil.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -26,6 +26,7 @@ import (
 	"net"
 	"os"
 	ex "os/exec"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"time"
@@ -36,6 +37,7 @@ import (
 	"github.com/cenkalti/backoff"
 	conf "github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/util"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/sirupsen/logrus"
 )
 
@@ -269,27 +271,35 @@ func sshExecExternal(c conf.ServerInfo, cmd string, sudo bool) (result execResul
 		return sshExecNative(c, cmd, sudo)
 	}
 
-	defaultSSHArgs := []string{
-		"-tt",
-		"-o", "StrictHostKeyChecking=yes",
-		"-o", "LogLevel=quiet",
-		"-o", "ConnectionAttempts=3",
-		"-o", "ConnectTimeout=10",
-		"-o", "ControlMaster=no",
-		"-o", "ControlPath=none",
+	defaultSSHArgs := []string{"-tt"}
 
-		// TODO ssh session multiplexing
-		//  "-o", "ControlMaster=auto",
-		//  "-o", `ControlPath=~/.ssh/controlmaster-%r-%h.%p`,
-		//  "-o", "Controlpersist=30m",
+	if !conf.Conf.SSHConfig {
+		home, err := homedir.Dir()
+		if err != nil {
+			msg := fmt.Sprintf("Failed to get HOME directory: %s", err)
+			result.Stderr = msg
+			result.ExitStatus = 997
+			return
+		}
+		controlPath := filepath.Join(home, ".vuls", `controlmaster-%r-`+c.ServerName+`.%p`)
+
+		defaultSSHArgs = append(defaultSSHArgs,
+			"-o", "StrictHostKeyChecking=yes",
+			"-o", "LogLevel=quiet",
+			"-o", "ConnectionAttempts=3",
+			"-o", "ConnectTimeout=10",
+			"-o", "ControlMaster=auto",
+			"-o", fmt.Sprintf("ControlPath=%s", controlPath),
+			"-o", "Controlpersist=10m",
+		)
 	}
+
+	if conf.Conf.Vvv {
+		defaultSSHArgs = append(defaultSSHArgs, "-vvv")
+	}
+
 	args := append(defaultSSHArgs, fmt.Sprintf("%s@%s", c.User, c.Host))
 	args = append(args, "-p", c.Port)
-
-	//  if conf.Conf.Debug {
-	//      args = append(args, "-v")
-	//  }
-
 	if 0 < len(c.KeyPath) {
 		args = append(args, "-i", c.KeyPath)
 		args = append(args, "-o", "PasswordAuthentication=no")
@@ -360,13 +370,21 @@ func decorateCmd(c conf.ServerInfo, cmd string, sudo bool) string {
 	//  }
 
 	if c.IsContainer() {
-		switch c.Containers.Type {
+		switch c.ContainerType {
 		case "", "docker":
 			cmd = fmt.Sprintf(`docker exec --user 0 %s %s -c '%s'`,
 				c.Container.ContainerID, dockerShell(c.Distro.Family), cmd)
 		case "lxd":
+			// If the user belong to the "lxd" group, root privilege is not required.
 			cmd = fmt.Sprintf(`lxc exec %s -- %s -c '%s'`,
 				c.Container.Name, dockerShell(c.Distro.Family), cmd)
+		case "lxc":
+			cmd = fmt.Sprintf(`lxc-attach -n %s 2>/dev/null -- %s -c '%s'`,
+				c.Container.Name, dockerShell(c.Distro.Family), cmd)
+			// LXC required root privilege
+			if c.User != "root" {
+				cmd = fmt.Sprintf("sudo -S %s", cmd)
+			}
 		}
 	}
 	//  cmd = fmt.Sprintf("set -x; %s", cmd)

scan/executil_test.go

@@ -1,5 +1,5 @@
 /* Vuls - Vulnerability Scanner
-Copyright (C) 2016  Future Architect, Inc. Japan.
+Copyright (C) 2016  Future Corporation , Japan.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -46,21 +46,21 @@ func TestDecorateCmd(t *testing.T) {
 		},
 		// non-root sudo false
 		{
-			conf:     config.ServerInfo{User: "non-roor"},
+			conf:     config.ServerInfo{User: "non-root"},
 			cmd:      "ls",
 			sudo:     false,
 			expected: "ls",
 		},
 		// non-root sudo true
 		{
-			conf:     config.ServerInfo{User: "non-roor"},
+			conf:     config.ServerInfo{User: "non-root"},
 			cmd:      "ls",
 			sudo:     true,
 			expected: "sudo -S ls",
 		},
 		// non-root sudo true
 		{
-			conf:     config.ServerInfo{User: "non-roor"},
+			conf:     config.ServerInfo{User: "non-root"},
 			cmd:      "ls | grep hoge",
 			sudo:     true,
 			expected: "sudo -S ls | grep hoge",
@@ -69,9 +69,9 @@ func TestDecorateCmd(t *testing.T) {
 		// root sudo false docker
 		{
 			conf: config.ServerInfo{
-				User:       "root",
-				Container:  config.Container{ContainerID: "abc"},
-				Containers: config.Containers{Type: "docker"},
+				User:          "root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "docker",
 			},
 			cmd:      "ls",
 			sudo:     false,
@@ -80,9 +80,9 @@ func TestDecorateCmd(t *testing.T) {
 		// root sudo true docker
 		{
 			conf: config.ServerInfo{
-				User:       "root",
-				Container:  config.Container{ContainerID: "abc"},
-				Containers: config.Containers{Type: "docker"},
+				User:          "root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "docker",
 			},
 			cmd:      "ls",
 			sudo:     true,
@@ -91,9 +91,9 @@ func TestDecorateCmd(t *testing.T) {
 		// non-root sudo false, docker
 		{
 			conf: config.ServerInfo{
-				User:       "non-root",
-				Container:  config.Container{ContainerID: "abc"},
-				Containers: config.Containers{Type: "docker"},
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "docker",
 			},
 			cmd:      "ls",
 			sudo:     false,
@@ -102,9 +102,9 @@ func TestDecorateCmd(t *testing.T) {
 		// non-root sudo true, docker
 		{
 			conf: config.ServerInfo{
-				User:       "non-root",
-				Container:  config.Container{ContainerID: "abc"},
-				Containers: config.Containers{Type: "docker"},
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "docker",
 			},
 			cmd:      "ls",
 			sudo:     true,
@@ -113,9 +113,9 @@ func TestDecorateCmd(t *testing.T) {
 		// non-root sudo true, docker
 		{
 			conf: config.ServerInfo{
-				User:       "non-root",
-				Container:  config.Container{ContainerID: "abc"},
-				Containers: config.Containers{Type: "docker"},
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "docker",
 			},
 			cmd:      "ls | grep hoge",
 			sudo:     true,
@@ -125,9 +125,9 @@ func TestDecorateCmd(t *testing.T) {
 		// root sudo false lxd
 		{
 			conf: config.ServerInfo{
-				User:       "root",
-				Container:  config.Container{ContainerID: "abc", Name: "def"},
-				Containers: config.Containers{Type: "lxd"},
+				User:          "root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxd",
 			},
 			cmd:      "ls",
 			sudo:     false,
@@ -136,9 +136,9 @@ func TestDecorateCmd(t *testing.T) {
 		// root sudo true lxd
 		{
 			conf: config.ServerInfo{
-				User:       "root",
-				Container:  config.Container{ContainerID: "abc", Name: "def"},
-				Containers: config.Containers{Type: "lxd"},
+				User:          "root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxd",
 			},
 			cmd:      "ls",
 			sudo:     true,
@@ -147,9 +147,9 @@ func TestDecorateCmd(t *testing.T) {
 		// non-root sudo false, lxd
 		{
 			conf: config.ServerInfo{
-				User:       "non-root",
-				Container:  config.Container{ContainerID: "abc", Name: "def"},
-				Containers: config.Containers{Type: "lxd"},
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxd",
 			},
 			cmd:      "ls",
 			sudo:     false,
@@ -158,9 +158,9 @@ func TestDecorateCmd(t *testing.T) {
 		// non-root sudo true, lxd
 		{
 			conf: config.ServerInfo{
-				User:       "non-root",
-				Container:  config.Container{ContainerID: "abc", Name: "def"},
-				Containers: config.Containers{Type: "lxd"},
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxd",
 			},
 			cmd:      "ls",
 			sudo:     true,
@@ -169,14 +169,70 @@ func TestDecorateCmd(t *testing.T) {
 		// non-root sudo true lxd
 		{
 			conf: config.ServerInfo{
-				User:       "non-root",
-				Container:  config.Container{ContainerID: "abc", Name: "def"},
-				Containers: config.Containers{Type: "lxd"},
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxd",
 			},
 			cmd:      "ls | grep hoge",
 			sudo:     true,
 			expected: `lxc exec def -- /bin/sh -c 'ls | grep hoge'`,
 		},
+		// -------------lxc-------------
+		// root sudo false lxc
+		{
+			conf: config.ServerInfo{
+				User:          "root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxc",
+			},
+			cmd:      "ls",
+			sudo:     false,
+			expected: `lxc-attach -n def 2>/dev/null -- /bin/sh -c 'ls'`,
+		},
+		// root sudo true lxc
+		{
+			conf: config.ServerInfo{
+				User:          "root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxc",
+			},
+			cmd:      "ls",
+			sudo:     true,
+			expected: `lxc-attach -n def 2>/dev/null -- /bin/sh -c 'ls'`,
+		},
+		// non-root sudo false, lxc
+		{
+			conf: config.ServerInfo{
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxc",
+			},
+			cmd:      "ls",
+			sudo:     false,
+			expected: `sudo -S lxc-attach -n def 2>/dev/null -- /bin/sh -c 'ls'`,
+		},
+		// non-root sudo true, lxc
+		{
+			conf: config.ServerInfo{
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxc",
+			},
+			cmd:      "ls",
+			sudo:     true,
+			expected: `sudo -S lxc-attach -n def 2>/dev/null -- /bin/sh -c 'ls'`,
+		},
+		// non-root sudo true lxc
+		{
+			conf: config.ServerInfo{
+				User:          "non-root",
+				Container:     config.Container{ContainerID: "abc", Name: "def"},
+				ContainerType: "lxc",
+			},
+			cmd:      "ls | grep hoge",
+			sudo:     true,
+			expected: `sudo -S lxc-attach -n def 2>/dev/null -- /bin/sh -c 'ls | grep hoge'`,
+		},
 	}
 
 	for _, tt := range tests {