add db backend redis (#445)

This commit is contained in:
sadayuki-matsuno
2017-06-27 18:10:09 +09:00
committed by kota kanbe
parent c442a433b0
commit 7778783dd8
16 changed files with 282 additions and 152 deletions

46
Gopkg.lock generated
View File

@@ -1,4 +1,5 @@
memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
@@ -13,10 +14,10 @@ memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
version = "v8.0.0"
[[projects]]
branch = "master"
name = "github.com/BurntSushi/toml"
packages = ["."]
revision = "b26d9c308763d68093482582cea63d69be07a0f0"
version = "v0.3.0"
revision = "8b58b6030fce084b58a61e2bc3fdf183d5881ab4"
[[projects]]
branch = "master"
@@ -65,6 +66,12 @@ memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
revision = "e7fea39b01aea8d5671f6858f0532f56e8bff3a5"
version = "v1.27.0"
[[projects]]
name = "github.com/go-redis/redis"
packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto"]
revision = "e14976b254c5bc5f399dd0ae9314b1d02a176897"
version = "v6.5.0"
[[projects]]
name = "github.com/go-sql-driver/mysql"
packages = ["."]
@@ -126,10 +133,9 @@ memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
revision = "9865fe14d09b1c729188ac810466dde90f897ee3"
[[projects]]
branch = "master"
name = "github.com/kotakanbe/go-cve-dictionary"
packages = ["config","db","jvn","log","models","nvd","util"]
revision = "d47709be4cc24d2c77a7be9096dcfcf211ba1d57"
revision = "c57d73c89e4d1a71f417ffcef6e13978a5add7ac"
[[projects]]
name = "github.com/kotakanbe/go-pingscanner"
@@ -138,10 +144,9 @@ memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
version = "v0.1.0"
[[projects]]
branch = "improve-db"
name = "github.com/kotakanbe/goval-dictionary"
packages = ["config","db","log","models"]
revision = "5f7aa97d45d565eaccc70c0c365e21624a9c6e3f"
packages = ["config","db","db/rdb","log","models"]
revision = "233459d2cc9ae85d8fcfb6a3d1412fdba6b0ea65"
[[projects]]
branch = "master"
@@ -149,18 +154,18 @@ memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
packages = ["."]
revision = "e7519b8c80ba008a3bfc57ffa31232bf2a77f455"
[[projects]]
branch = "master"
name = "github.com/lib/pq"
packages = [".","hstore","oid"]
revision = "2704adc878c21e1329f46f6e56a1c387d788ff94"
[[projects]]
name = "github.com/labstack/gommon"
packages = ["color","log"]
revision = "1121fd3e243c202482226a7afe4dcd07ffc4139a"
version = "v0.2.1"
[[projects]]
branch = "master"
name = "github.com/lib/pq"
packages = [".","hstore","oid"]
revision = "8837942c3e09574accbc5f150e2c5e057189cace"
[[projects]]
name = "github.com/mattn/go-colorable"
packages = ["."]
@@ -245,6 +250,12 @@ memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
packages = ["oval"]
revision = "003ac9af5fffac6c97ab1def025d2cb73e88469a"
[[projects]]
branch = "master"
name = "go4.org"
packages = ["syncutil"]
revision = "034d17a462f7b2dcd1a4a73553ec5357ff6e6c6e"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
@@ -268,3 +279,10 @@ memo = "e59ec63c1c329674a0e5e4236131c787e5b81bab37529104fdc02ed8fdf29283"
name = "golang.org/x/text"
packages = ["internal/gen","internal/triegen","internal/ucd","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
revision = "19e51611da83d6be54ddafce4a4af510cb3e9ea4"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "2f4b3e869b7567e51d9bff86e52b3960d8c8304164b40d69f03a9e690032c9f5"
solver-name = "gps-cdcl"
solver-version = 1

View File

@@ -1,36 +1,36 @@
[[dependencies]]
[[constraint]]
branch = "master"
name = "github.com/Azure/azure-storage-go"
[[dependencies]]
[[constraint]]
branch = "master"
name = "github.com/BurntSushi/toml"
[[dependencies]]
[[constraint]]
branch = "master"
name = "github.com/Sirupsen/logrus"
[[dependencies]]
[[constraint]]
name = "github.com/aws/aws-sdk-go"
revision = "5b341290c488aa6bd76b335d819b4a68516ec3ab"
[[dependencies]]
[[constraint]]
branch = "master"
name = "github.com/jroimartin/gocui"
[[dependencies]]
[[constraint]]
branch = "master"
name = "github.com/k0kubun/pp"
[[dependencies]]
branch = "master"
[[constraint]]
name = "github.com/kotakanbe/go-cve-dictionary"
revision = "c57d73c89e4d1a71f417ffcef6e13978a5add7ac"
[[dependencies]]
branch = "improve-db"
[[constraint]]
revision = "233459d2cc9ae85d8fcfb6a3d1412fdba6b0ea65"
name = "github.com/kotakanbe/goval-dictionary"
[[dependencies]]
[[constraint]]
branch = "master"
name = "github.com/kotakanbe/logrus-prefixed-formatter"

View File

@@ -82,6 +82,7 @@ Slackチームは[こちらから](http://goo.gl/forms/xm5KFo35tu)参加でき
* [Example: Add optional key-value pairs to JSON](#example-add-optional-key-value-pairs-to-json)
* [Example: Use MySQL as a DB storage back-end](#example-use-mysql-as-a-db-storage-back-end)
* [Example: Use PostgreSQL as a DB storage back-end](#example-use-postgresql-as-a-db-storage-back-end)
* [Example: Use Redis as a DB storage back-end](#example-use-redis-as-a-db-storage-back-end)
- [Usage: Scan vulnerability of non-OS package](#usage-scan-vulnerability-of-non-os-package)
- [Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental)](#usage-integrate-with-owasp-dependency-check-to-automatic-update-when-the-libraries-are-updated-experimental)
- [Usage: TUI](#usage-tui)
@@ -194,7 +195,7 @@ Hello Vulsチュートリアルでは手動でのセットアップ方法で説
Vulsセットアップに必要な以下のソフトウェアをインストールする。
- SQLite3 or MySQL
- SQLite3, MySQL, PostgreSQL or Redis
- git
- gcc
- GNU Make
@@ -1041,7 +1042,7 @@ report:
[-results-dir=/path/to/results]
[-log-dir=/path/to/log]
[-refresh-cve]
[-cvedb-type=sqlite3|mysql|postgres]
[-cvedb-type=sqlite3|mysql|postgres|redis]
[-cvedb-path=/path/to/cve.sqlite3]
[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
[-cvss-over=7]
@@ -1088,7 +1089,7 @@ report:
-cvedb-path string
/path/to/sqlite3 (For get cve detail from cve.sqlite3)
-cvedb-type string
DB type for fetching CVE dictionary (sqlite3, mysql or postgres) (default "sqlite3")
DB type for fetching CVE dictionary (sqlite3, mysql, postgres or redis) (default "sqlite3")
-cvedb-url string
http://cve-dictionary.com:8080 or DB connection string
-cvss-over float
@@ -1436,6 +1437,14 @@ $ vuls report \
-cvedb-url=""host=myhost user=user dbname=dbname sslmode=disable password=password""
```
## Example: Use Redis as a DB storage back-end
```
$ vuls report \
-cvedb-type=redis -cvedb-url="redis://localhost/0"
-ovaldb-type=redis -ovaldb-url="redis://localhost/1"
```
----
# Usage: Scan vulnerability of non-OS package
@@ -1496,7 +1505,7 @@ VulsとDependency Checkを連携すると以下の利点がある
```
tui:
tui
[-cvedb-type=sqlite3|mysql|postgres]
[-cvedb-type=sqlite3|mysql|postgres|redis]
[-cvedb-path=/path/to/cve.sqlite3]
[-cvedb-url=http://127.0.0.1:1323 DB connection string]
[-refresh-cve]
@@ -1509,7 +1518,7 @@ tui:
-cvedb-path string
/path/to/sqlite3 (For get cve detail from cve.sqlite3)
-cvedb-type string
DB type for fetching CVE dictionary (sqlite3, mysql or postgres) (default "sqlite3")
DB type for fetching CVE dictionary (sqlite3, mysql, postgres or redis) (default "sqlite3")
-cvedb-url string
http://cve-dictionary.com:8080 or DB connection string
-debug
@@ -1621,7 +1630,7 @@ slack, emailは日本語対応済み TUIは日本語表示未対応
# Update Vuls With Glide
- Update go-cve-dictionary
If the DB schema was changed, please specify new SQLite3 or MySQL DB file.
If the DB schema was changed, please specify new SQLite3, MySQL, PostgreSQL or Redis DB file.
```
$ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary
$ git pull

View File

@@ -89,6 +89,7 @@ We have a slack team. [Join slack team](http://goo.gl/forms/xm5KFo35tu)
* [Example: Add optional key-value pairs to JSON](#example-add-optional-key-value-pairs-to-json)
* [Example: Use MySQL as a DB storage back-end](#example-use-mysql-as-a-db-storage-back-end)
* [Example: Use PostgreSQL as a DB storage back-end](#example-use-postgresql-as-a-db-storage-back-end)
* [Example: Use Redis as a DB storage back-end](#example-use-redis-as-a-db-storage-back-end)
- [Usage: Scan vulnerabilites of non-OS packages](#usage-scan-vulnerabilites-of-non-os-packages)
- [Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental)](#usage-integrate-with-owasp-dependency-check-to-automatic-update-when-the-libraries-are-updated-experimental)
- [Usage: TUI](#usage-tui)
@@ -199,7 +200,7 @@ This can be done in the following steps.
Vuls requires the following packages.
- SQLite3 or MySQL
- SQLite3, MySQL, PostgreSQL, Redis
- git
- gcc
- GNU Make
@@ -504,7 +505,7 @@ On the aggregation server, you can refer to the scanning result of each scan tar
[Details](#example-scan-via-shell-instead-of-ssh)
## [go-cve-dictionary](https://github.com/kotakanbe/go-cve-dictionary)
- Fetch vulnerability information from NVD and JVN(Japanese), then insert into SQLite3 or MySQL.
- Fetch vulnerability information from NVD and JVN(Japanese), then insert into SQLite3, MySQL, PostgreSQL or Redis.
## Scanning Flow
![Vuls-Scan-Flow](img/vuls-scan-flow.png)
@@ -1438,6 +1439,14 @@ $ vuls report \
-cvedb-url=""host=myhost user=user dbname=dbname sslmode=disable password=password""
```
## Example: Use Redis as a DB storage back-end
```
$ vuls report \
-cvedb-type=redis -cvedb-url="redis://localhost/0"
-ovaldb-type=redis -ovaldb-url="redis://localhost/1"
```
----
# Usage: Scan vulnerabilites of non-OS packages
@@ -1583,7 +1592,7 @@ see [go-cve-dictionary#usage-fetch-nvd-data](https://github.com/kotakanbe/go-cve
# How to Update
- Update go-cve-dictionary
If the DB schema was changed, please specify new SQLite3 or MySQL DB file.
If the DB schema was changed, please specify new SQLite3, MySQL, PostgreSQL or Redis DB file.
```
$ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary
$ git pull

View File

@@ -19,6 +19,7 @@ package config
import (
"fmt"
"os"
"runtime"
"strconv"
"strings"
@@ -30,6 +31,35 @@ import (
// Conf has Configuration
var Conf Config
const (
// RedHat is
RedHat = "redhat"
// Debian is
Debian = "debian"
// Ubuntu is
Ubuntu = "ubuntu"
// CentOS is
CentOS = "centos"
// Fedora is
Fedora = "fedora"
// Amazon is
Amazon = "amazon"
// Oracle is
Oracle = "oracle"
// FreeBSD is
FreeBSD = "freebsd"
// Raspbian is
Raspbian = "raspbian"
)
//Config is struct of Configuration
type Config struct {
Debug bool
@@ -163,26 +193,12 @@ func (c Config) ValidateOnReport() bool {
}
}
switch c.CveDBType {
case "sqlite3":
if ok, _ := valid.IsFilePath(c.CveDBPath); !ok {
errs = append(errs, fmt.Errorf(
"SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cvedb-path: %s",
c.CveDBPath))
}
case "mysql":
if c.CveDBURL == "" {
errs = append(errs, fmt.Errorf(
`MySQL connection string is needed. -cvedb-url="user:pass@tcp(localhost:3306)/dbname"`))
}
case "postgres":
if c.CveDBURL == "" {
errs = append(errs, fmt.Errorf(
`PostgreSQL connection string is needed. -cvedb-url=""host=myhost user=user dbname=dbname sslmode=disable password=password""`))
}
default:
errs = append(errs, fmt.Errorf(
"CVE DB type must be either 'sqlite3', 'mysql' or 'postgres'. -cvedb-type: %s", c.CveDBType))
if err := validateDB("cvedb", c.CveDBType, c.CveDBPath, c.CveDBURL); err != nil {
errs = append(errs, err)
}
if err := validateDB("ovaldb", c.OvalDBType, c.OvalDBPath, c.OvalDBURL); err != nil {
errs = append(errs, err)
}
_, err := valid.ValidateStruct(c)
@@ -216,16 +232,8 @@ func (c Config) ValidateOnTui() bool {
}
}
if c.CveDBType != "sqlite3" && c.CveDBType != "mysql" && c.CveDBType != "postgres" {
errs = append(errs, fmt.Errorf(
"CVE DB type must be either 'sqlite3', 'mysql' or 'postgres'. -cve-dictionary-dbtype: %s", c.CveDBType))
}
if c.CveDBType == "sqlite3" {
if ok, _ := valid.IsFilePath(c.CveDBPath); !ok {
errs = append(errs, fmt.Errorf(
"SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath))
}
if err := validateDB("cvedb", c.CveDBType, c.CveDBPath, c.CveDBURL); err != nil {
errs = append(errs, err)
}
for _, err := range errs {
@@ -235,6 +243,51 @@ func (c Config) ValidateOnTui() bool {
return len(errs) == 0
}
// validateDB validates configuration
// dictionaryDB name is 'cvedb' or 'ovaldb'
func validateDB(dictionaryDBName, dbType, dbPath, dbURL string) error {
switch dbType {
case "sqlite3":
if ok, _ := valid.IsFilePath(dbPath); !ok {
return fmt.Errorf(
"SQLite3 DB path (%s) must be a *Absolute* file path. -%s-path: %s",
dictionaryDBName,
dictionaryDBName,
dbPath)
}
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
return fmt.Errorf("SQLite3 DB path (%s) is not exist: %s",
dictionaryDBName,
dbPath)
}
case "mysql":
if dbURL == "" {
return fmt.Errorf(
`MySQL connection string is needed. -%s-url="user:pass@tcp(localhost:3306)/dbname"`,
dictionaryDBName)
}
case "postgres":
if dbURL == "" {
return fmt.Errorf(
`PostgreSQL connection string is needed. -%s-url="host=myhost user=user dbname=dbname sslmode=disable password=password"`,
dictionaryDBName)
}
case "redis":
if dbURL == "" {
return fmt.Errorf(
`Redis connection string is needed. -%s-url="redis://localhost/0"`,
dictionaryDBName)
}
default:
return fmt.Errorf(
"%s type must be either 'sqlite3', 'mysql', 'postgres' or 'redis'. -%s-type: %s",
dictionaryDBName,
dictionaryDBName,
dbType)
}
return nil
}
// SMTPConf is smtp config
type SMTPConf struct {
SMTPAddr string

View File

@@ -736,7 +736,7 @@ func TestVendorLink(t *testing.T) {
}{
{
in: in{
family: "rhel",
family: "redhat",
vinfo: VulnInfo{
CveID: "CVE-2017-6074",
CveContents: CveContents{

View File

@@ -126,33 +126,33 @@ func (v VulnInfo) Cvss3CalcURL() string {
func (v VulnInfo) VendorLinks(family string) map[string]string {
links := map[string]string{}
switch family {
case "rhel", "centos":
case config.RedHat, config.CentOS:
links["RHEL-CVE"] = "https://access.redhat.com/security/cve/" + v.CveID
for _, advisory := range v.DistroAdvisories {
aidURL := strings.Replace(advisory.AdvisoryID, ":", "-", -1)
links[advisory.AdvisoryID] = fmt.Sprintf("https://rhn.redhat.com/errata/%s.html", aidURL)
}
return links
case "oraclelinux":
case config.Oracle:
links["Oracle-CVE"] = fmt.Sprintf("https://linux.oracle.com/cve/%s.html", v.CveID)
for _, advisory := range v.DistroAdvisories {
links[advisory.AdvisoryID] =
fmt.Sprintf("https://linux.oracle.com/errata/%s.html", advisory.AdvisoryID)
}
return links
case "amazon":
case config.Amazon:
links["RHEL-CVE"] = "https://access.redhat.com/security/cve/" + v.CveID
for _, advisory := range v.DistroAdvisories {
links[advisory.AdvisoryID] =
fmt.Sprintf("https://alas.aws.amazon.com/%s.html", advisory.AdvisoryID)
}
return links
case "ubuntu":
case config.Ubuntu:
links["Ubuntu-CVE"] = "http://people.ubuntu.com/~ubuntu-security/cve/" + v.CveID
return links
case "debian":
case config.Debian:
links["Debian-CVE"] = "https://security-tracker.debian.org/tracker/" + v.CveID
case "FreeBSD":
case config.FreeBSD:
for _, advisory := range v.DistroAdvisories {
links["FreeBSD-VuXML"] = fmt.Sprintf("https://vuxml.freebsd.org/freebsd/%s.html", advisory.AdvisoryID)

View File

@@ -9,6 +9,7 @@ import (
ver "github.com/knqyf263/go-deb-version"
ovalconf "github.com/kotakanbe/goval-dictionary/config"
db "github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
ovalmodels "github.com/kotakanbe/goval-dictionary/models"
)
@@ -19,13 +20,27 @@ type DebianBase struct{ Base }
func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error {
ovalconf.Conf.DBType = config.Conf.OvalDBType
ovalconf.Conf.DBPath = config.Conf.OvalDBPath
if ovalconf.Conf.DBType == "sqlite3" {
ovalconf.Conf.DBPath = config.Conf.OvalDBPath
} else {
ovalconf.Conf.DBPath = config.Conf.OvalDBURL
}
util.Log.Infof("open oval-dictionary db (%s): %s",
config.Conf.OvalDBType, config.Conf.OvalDBPath)
ovalconf.Conf.DBType, ovalconf.Conf.DBPath)
ovaldb, err := db.NewDB(r.Family)
if err != nil {
ovallog.Initialize(config.Conf.LogDir)
var err error
var ovaldb db.DB
if ovaldb, err = db.NewDB(
ovalconf.Debian,
ovalconf.Conf.DBType,
ovalconf.Conf.DBPath,
ovalconf.Conf.DebugSQL,
); err != nil {
return err
}
defer ovaldb.CloseDB()
for _, pack := range r.Packages {
definitions, err := ovaldb.GetByPackName(r.Release, pack.Name)

View File

@@ -11,6 +11,7 @@ import (
ver "github.com/knqyf263/go-deb-version"
ovalconf "github.com/kotakanbe/goval-dictionary/config"
db "github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
ovalmodels "github.com/kotakanbe/goval-dictionary/models"
)
@@ -57,14 +58,28 @@ func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string,
packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) {
ovalconf.Conf.DBType = config.Conf.OvalDBType
ovalconf.Conf.DBPath = config.Conf.OvalDBPath
if ovalconf.Conf.DBType == "sqlite3" {
ovalconf.Conf.DBPath = config.Conf.OvalDBPath
} else {
ovalconf.Conf.DBPath = config.Conf.OvalDBURL
}
util.Log.Infof("open oval-dictionary db (%s): %s",
config.Conf.OvalDBType, config.Conf.OvalDBPath)
ovalconf.Conf.DBType, ovalconf.Conf.DBPath)
d := db.NewRedHat()
defer d.Close()
ovallog.Initialize(config.Conf.LogDir)
var ovaldb db.DB
if ovaldb, err = db.NewDB(
ovalconf.RedHat,
ovalconf.Conf.DBType,
ovalconf.Conf.DBPath,
ovalconf.Conf.DebugSQL,
); err != nil {
return
}
defer ovaldb.CloseDB()
for _, pack := range packs {
definitions, err := d.GetByPackName(osRelease, pack.Name)
definitions, err := ovaldb.GetByPackName(osRelease, pack.Name)
if err != nil {
return nil, fmt.Errorf("Failed to get RedHat OVAL info by package name: %v", err)
}

View File

@@ -26,6 +26,7 @@ import (
"github.com/cenkalti/backoff"
"github.com/parnurzeal/gorequest"
log "github.com/Sirupsen/logrus"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/util"
cveconfig "github.com/kotakanbe/go-cve-dictionary/config"
@@ -69,7 +70,7 @@ type response struct {
CveDetail cve.CveDetail
}
func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDetails, err error) {
func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails []*cve.CveDetail, err error) {
if !api.isFetchViaHTTP() {
return api.FetchCveDetailsFromCveDB(cveIDs)
}
@@ -111,26 +112,26 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet
select {
case res := <-resChan:
if len(res.CveDetail.CveID) == 0 {
cveDetails = append(cveDetails, cve.CveDetail{
cveDetails = append(cveDetails, &cve.CveDetail{
CveID: res.Key,
})
} else {
cveDetails = append(cveDetails, res.CveDetail)
cveDetails = append(cveDetails, &res.CveDetail)
}
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return []cve.CveDetail{}, fmt.Errorf("Timeout Fetching CVE")
return []*cve.CveDetail{}, fmt.Errorf("Timeout Fetching CVE")
}
}
if len(errs) != 0 {
return []cve.CveDetail{},
return []*cve.CveDetail{},
fmt.Errorf("Failed to fetch CVE. err: %v", errs)
}
return
}
func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails cve.CveDetails, err error) {
func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails []*cve.CveDetail, err error) {
util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
cveconfig.Conf.DBType = config.Conf.CveDBType
if config.Conf.CveDBType == "sqlite3" {
@@ -139,14 +140,27 @@ func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails c
cveconfig.Conf.DBPath = config.Conf.CveDBURL
}
cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
if err := cvedb.OpenDB(); err != nil {
return []cve.CveDetail{},
var driver cvedb.DB
if driver, err = cvedb.NewDB(cveconfig.Conf.DBType); err != nil {
log.Error(err)
return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err)
}
log.Infof("Opening DB (%s).", driver.Name())
if err := driver.OpenDB(
cveconfig.Conf.DBType,
cveconfig.Conf.DBPath,
cveconfig.Conf.DebugSQL,
); err != nil {
return []*cve.CveDetail{},
fmt.Errorf("Failed to open DB. err: %s", err)
}
for _, cveID := range cveIDs {
cveDetail := cvedb.Get(cveID)
cveDetail := driver.Get(cveID)
if len(cveDetail.CveID) == 0 {
cveDetails = append(cveDetails, cve.CveDetail{
cveDetails = append(cveDetails, &cve.CveDetail{
CveID: cveID,
})
} else {
@@ -203,12 +217,12 @@ func (api cvedictClient) isFetchViaHTTP() bool {
return false
}
func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]cve.CveDetail, error) {
func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]*cve.CveDetail, error) {
if api.isFetchViaHTTP() {
api.baseURL = config.Conf.CveDBURL
url, err := util.URLPathJoin(api.baseURL, "cpes")
if err != nil {
return []cve.CveDetail{}, err
return []*cve.CveDetail{}, err
}
query := map[string]string{"name": cpeName}
@@ -219,7 +233,7 @@ func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]cve.CveDeta
return api.FetchCveDetailsByCpeNameFromDB(cpeName)
}
func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cve.CveDetail, error) {
func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]*cve.CveDetail, error) {
var body string
var errs []error
var resp *http.Response
@@ -240,18 +254,18 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
return []cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err)
return []*cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err)
}
cveDetails := []cve.CveDetail{}
cveDetails := []*cve.CveDetail{}
if err := json.Unmarshal([]byte(body), &cveDetails); err != nil {
return []cve.CveDetail{},
return []*cve.CveDetail{},
fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
}
return cveDetails, nil
}
func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.CveDetail, error) {
func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) (cveDetails []*cve.CveDetail, err error) {
util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
cveconfig.Conf.DBType = config.Conf.CveDBType
if config.Conf.CveDBType == "sqlite3" {
@@ -261,9 +275,20 @@ func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.C
}
cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
if err := cvedb.OpenDB(); err != nil {
return []cve.CveDetail{},
var driver cvedb.DB
if driver, err = cvedb.NewDB(cveconfig.Conf.DBType); err != nil {
log.Error(err)
return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err)
}
log.Infof("Opening DB (%s).", driver.Name())
if err = driver.OpenDB(
cveconfig.Conf.DBType,
cveconfig.Conf.DBPath,
cveconfig.Conf.DebugSQL,
); err != nil {
return []*cve.CveDetail{},
fmt.Errorf("Failed to open DB. err: %s", err)
}
return cvedb.GetByCpeName(cpeName), nil
return driver.GetByCpeName(cpeName), nil
}

View File

@@ -19,7 +19,6 @@ package report
import (
"fmt"
"os"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
@@ -79,19 +78,6 @@ func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, erro
func fillCveInfo(r *models.ScanResult) error {
util.Log.Debugf("need to refresh")
if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" {
if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) {
return fmt.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s",
c.Conf.CveDBPath)
}
}
if c.Conf.OvalDBType == "sqlite3" && c.Conf.OvalDBURL == "" {
if _, err := os.Stat(c.Conf.OvalDBPath); os.IsNotExist(err) {
// TODO Warning??
return fmt.Errorf("SQLite3 DB(OVAL-Dictionary) is not exist: %s",
c.Conf.OvalDBPath)
}
}
util.Log.Debugf("Fill CVE detailed information with OVAL")
if err := fillWithOval(r); err != nil {
@@ -156,15 +142,15 @@ func fillWithCveDB(r *models.ScanResult) error {
func fillWithOval(r *models.ScanResult) error {
var ovalClient oval.Client
switch r.Family {
case "debian":
case c.Debian:
ovalClient = oval.NewDebian()
case "ubuntu":
case c.Ubuntu:
ovalClient = oval.NewUbuntu()
case "rhel":
case c.RedHat:
ovalClient = oval.NewRedhat()
case "centos":
case c.CentOS:
ovalClient = oval.NewCentOS()
case "amazon", "oraclelinux", "Raspbian", "FreeBSD":
case c.Amazon, c.Oracle, c.Raspbian, c.FreeBSD:
//TODO implement OracleLinux
return nil
default:

View File

@@ -73,7 +73,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
// e.g.
// Raspbian GNU/Linux 7 \n \l
result := strings.Fields(r.Stdout)
if len(result) > 2 && result[0] == "Raspbian" {
if len(result) > 2 && result[0] == config.Raspbian {
distro := strings.ToLower(trim(result[0]))
deb.setDistro(distro, trim(result[2]))
return true, deb, nil
@@ -121,7 +121,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
// Debian
cmd := "cat /etc/debian_version"
if r := exec(c, cmd, noSudo); r.isSuccess() {
deb.setDistro("debian", trim(r.Stdout))
deb.setDistro(config.Debian, trim(r.Stdout))
return true, deb, nil
}
@@ -147,10 +147,10 @@ func (o *debian) checkIfSudoNoPasswd() error {
func (o *debian) checkDependencies() error {
switch o.Distro.Family {
case "ubuntu", "raspbian":
case config.Ubuntu, config.Raspbian:
return nil
case "debian":
case config.Debian:
// Debian needs aptitude to get changelogs.
// Because unable to get changelogs via apt-get changelog on Debian.
if r := o.exec("test -f /usr/bin/aptitude", noSudo); !r.isSuccess() {
@@ -539,9 +539,9 @@ func (o *debian) getChangelogCache(meta *cache.Meta, pack models.Package) string
func (o *debian) scanPackageCveIDs(pack models.Package) ([]DetectedCveID, *models.Package, error) {
cmd := ""
switch o.Distro.Family {
case "ubuntu", "raspbian":
case config.Ubuntu, config.Raspbian:
cmd = fmt.Sprintf(`PAGER=cat apt-get -q=2 changelog %s`, pack.Name)
case "debian":
case config.Debian:
cmd = fmt.Sprintf(`PAGER=cat aptitude -q=2 changelog %s`, pack.Name)
}
cmd = util.PrependProxyEnv(cmd)
@@ -592,10 +592,10 @@ func (o *debian) getCveIDsFromChangelog(
delim := []string{"+", "~", "build"}
switch o.Distro.Family {
case "ubuntu":
delim = append(delim, "ubuntu")
case "debian":
case "Raspbian":
case config.Ubuntu:
delim = append(delim, config.Ubuntu)
case config.Debian:
case config.Raspbian:
}
for _, d := range delim {

View File

@@ -167,7 +167,7 @@ func exec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (resul
func localExec(c conf.ServerInfo, cmdstr string, sudo bool) (result execResult) {
cmdstr = decorateCmd(c, cmdstr, sudo)
var cmd *ex.Cmd
if c.Distro.Family == "FreeBSD" {
if c.Distro.Family == conf.FreeBSD {
cmd = ex.Command("/bin/sh", "-c", cmdstr)
} else {
cmd = ex.Command("/bin/bash", "-c", cmdstr)

View File

@@ -51,13 +51,13 @@ func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) {
bsd = newBsd(c)
// Prevent from adding `set -o pipefail` option
c.Distro = config.Distro{Family: "FreeBSD"}
c.Distro = config.Distro{Family: config.FreeBSD}
if r := exec(c, "uname", noSudo); r.isSuccess() {
if strings.Contains(r.Stdout, "FreeBSD") == true {
if strings.Contains(r.Stdout, config.FreeBSD) == true {
if b := exec(c, "freebsd-version", noSudo); b.isSuccess() {
rel := strings.TrimSpace(b.Stdout)
bsd.setDistro("FreeBSD", rel)
bsd.setDistro(config.FreeBSD, rel)
return true, bsd
}
}

View File

@@ -55,7 +55,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) {
red = newRedhat(c)
if r := exec(c, "ls /etc/fedora-release", noSudo); r.isSuccess() {
red.setDistro("fedora", "unknown")
red.setDistro(config.Fedora, "unknown")
util.Log.Warn("Fedora not tested yet: %s", r)
return true, red
}
@@ -72,7 +72,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) {
}
release := result[2]
red.setDistro("oraclelinux", release)
red.setDistro(config.Oracle, release)
return true, red
}
}
@@ -93,9 +93,9 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) {
release := result[2]
switch strings.ToLower(result[1]) {
case "centos", "centos linux":
red.setDistro("centos", release)
red.setDistro(config.CentOS, release)
default:
red.setDistro("rhel", release)
red.setDistro(config.RedHat, release)
}
return true, red
}
@@ -103,7 +103,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) {
}
if r := exec(c, "ls /etc/system-release", noSudo); r.isSuccess() {
family := "amazon"
family := config.Amazon
release := "unknown"
if r := exec(c, "cat /etc/system-release", noSudo); r.isSuccess() {
fields := strings.Fields(r.Stdout)
@@ -133,12 +133,12 @@ func (o *redhat) checkIfSudoNoPasswd() error {
var zero = []int{0}
switch o.Distro.Family {
case "centos":
case config.CentOS:
cmds = []cmd{
{"yum --changelog --assumeno update yum", []int{0, 1}},
}
case "rhel", "oraclelinux":
case config.RedHat, config.Oracle:
majorVersion, err := o.Distro.MajorVersion()
if err != nil {
return fmt.Errorf("Not implemented yet: %s, err: %s", o.Distro, err)
@@ -180,7 +180,7 @@ func (o *redhat) checkIfSudoNoPasswd() error {
// Amazon ... -
func (o *redhat) checkDependencies() error {
var packName string
if o.Distro.Family == "amazon" {
if o.Distro.Family == config.Amazon {
return nil
}
@@ -191,7 +191,7 @@ func (o *redhat) checkDependencies() error {
return fmt.Errorf(msg)
}
if o.Distro.Family == "centos" {
if o.Distro.Family == config.CentOS {
if majorVersion < 6 {
msg := fmt.Sprintf("CentOS %s is not supported", o.Distro.Release)
o.log.Errorf(msg)
@@ -208,9 +208,9 @@ func (o *redhat) checkDependencies() error {
}
switch o.Distro.Family {
case "centos":
case config.CentOS:
packName = "yum-plugin-changelog"
case "rhel", "oraclelinux":
case config.RedHat, config.Oracle:
if majorVersion < 6 {
packName = "yum-security"
} else {
@@ -293,7 +293,7 @@ func (o *redhat) parseScannedPackagesLine(line string) (models.Package, error) {
}
func (o *redhat) scanVulnInfos() (models.VulnInfos, error) {
if o.Distro.Family != "centos" {
if o.Distro.Family != config.CentOS {
// Amazon, RHEL, Oracle Linux has yum updateinfo as default
// yum updateinfo can collenct vendor advisory information.
return o.scanUnsecurePackagesUsingYumPluginSecurity()
@@ -535,7 +535,7 @@ func (o *redhat) getChangelogCVELines(rpm2changelog map[string]*string, pack mod
func (o *redhat) divideChangelogByPackage(allChangelog string) (map[string]*string, error) {
var majorVersion int
var err error
if o.Distro.Family == "centos" {
if o.Distro.Family == config.CentOS {
majorVersion, err = o.Distro.MajorVersion()
if err != nil {
return nil, fmt.Errorf("Not implemented yet: %s, err: %s", o.Distro, err)
@@ -659,7 +659,7 @@ type distroAdvisoryCveIDs struct {
// Scaning unsecure packages using yum-plugin-security.
// Amazon, RHEL, Oracle Linux
func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, error) {
if o.Distro.Family == "centos" {
if o.Distro.Family == config.CentOS {
// CentOS has no security channel.
// So use yum check-update && parse changelog
return nil, fmt.Errorf(
@@ -678,7 +678,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos,
return nil, fmt.Errorf("Not implemented yet: %s, err: %s", o.Distro, err)
}
if (o.Distro.Family == "rhel" || o.Distro.Family == "oraclelinux") && major == 5 {
if (o.Distro.Family == config.RedHat || o.Distro.Family == config.Oracle) && major == 5 {
cmd = "yum --color=never list-security --security"
} else {
cmd = "yum --color=never --security updateinfo list updates"
@@ -721,7 +721,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos,
}
// get advisoryID(RHSA, ALAS, ELSA) - CVE IDs
if (o.Distro.Family == "rhel" || o.Distro.Family == "oraclelinux") && major == 5 {
if (o.Distro.Family == config.RedHat || o.Distro.Family == config.Oracle) && major == 5 {
cmd = "yum --color=never info-security"
} else {
cmd = "yum --color=never --security updateinfo updates"
@@ -817,12 +817,12 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID
switch sectionState {
case Header:
switch o.Distro.Family {
case "centos":
case config.CentOS:
// CentOS has no security channel.
// So use yum check-update && parse changelog
return result, fmt.Errorf(
"yum updateinfo is not suppported on CentOS")
case "rhel", "amazon", "oraclelinux":
case config.RedHat, config.Amazon, config.Oracle:
// nop
}
@@ -1032,7 +1032,7 @@ func (o *redhat) clone() osTypeInterface {
func (o *redhat) sudo() bool {
switch o.Distro.Family {
case "amazon":
case config.Amazon:
return false
default:
return true

View File

@@ -421,7 +421,7 @@ func setupChangelogCache() error {
needToSetupCache := false
for _, s := range servers {
switch s.getDistro().Family {
case "ubuntu", "debian", "raspbian":
case config.Ubuntu, config.Debian, config.Raspbian:
needToSetupCache = true
break
}