Enable to detect vulnerabilities on FreeBSD

This commit is contained in:
kota kanbe
2016-06-13 22:41:48 +09:00
parent c209564945
commit 17181405e3
10 changed files with 327 additions and 185 deletions

View File

@@ -221,10 +221,11 @@ type ServerInfo struct {
// Container Names or IDs
Containers []string
// userd internal
// used internal
LogMsgAnsiColor string // DebugLog Color
SudoOpt SudoOption
Container Container
Family string
}
// IsContainer returns whether this ServerInfo is about container

View File

@@ -305,7 +305,7 @@ func (p PackageInfo) ToStringNewVersion() string {
return str
}
// DistroAdvisory has Amazon Linux AMI Security Advisory information.
// DistroAdvisory has Amazon Linux, RHEL, FreeBSD Security Advisory information.
type DistroAdvisory struct {
gorm.Model `json:"-"`
CveInfoID uint `json:"-"`

View File

@@ -341,6 +341,15 @@ func distroLinks(cveInfo models.CveInfo, osFamily string) []distroLink {
},
// TODO Debian dsa
}
case "FreeBSD":
links := []distroLink{}
for _, advisory := range cveInfo.DistroAdvisories {
links = append(links, distroLink{
"FreeBSD-VuXML",
fmt.Sprintf(freeBSDVuXMLBaseURL, advisory.AdvisoryID),
})
}
return links
default:
return []distroLink{}
}

View File

@@ -31,6 +31,8 @@ const (
ubuntuSecurityBaseURL = "http://people.ubuntu.com/~ubuntu-security/cve"
debianTrackerBaseURL = "https://security-tracker.debian.org/tracker"
freeBSDVuXMLBaseURL = "https://vuxml.freebsd.org/freebsd/%s.html"
)
// ResultWriter Interface

View File

@@ -194,7 +194,7 @@ func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error)
lines := strings.Split(r.Stdout, "\n")
for _, line := range lines {
if trimmed := strings.TrimSpace(line); len(trimmed) != 0 {
name, version, err := o.parseScanedPackagesLine(trimmed)
name, version, err := o.parseScannedPackagesLine(trimmed)
if err != nil {
return nil, fmt.Errorf(
"Debian: Failed to parse package line: %s", line)
@@ -208,7 +208,7 @@ func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error)
return
}
func (o *debian) parseScanedPackagesLine(line string) (name, version string, err error) {
func (o *debian) parseScannedPackagesLine(line string) (name, version string, err error) {
re, _ := regexp.Compile(`^([^\t']+)\t(.+)$`)
result := re.FindStringSubmatch(line)
if len(result) == 3 {

View File

@@ -25,7 +25,7 @@ import (
"github.com/k0kubun/pp"
)
func TestParseScanedPackagesLineDebian(t *testing.T) {
func TestParseScannedPackagesLineDebian(t *testing.T) {
var packagetests = []struct {
in string
@@ -43,7 +43,7 @@ func TestParseScanedPackagesLineDebian(t *testing.T) {
d := newDebian(config.ServerInfo{})
for _, tt := range packagetests {
n, v, _ := d.parseScanedPackagesLine(tt.in)
n, v, _ := d.parseScannedPackagesLine(tt.in)
if n != tt.name {
t.Errorf("name: expected %s, actual %s", tt.name, n)
}

View File

@@ -2,9 +2,7 @@ package scan
import (
"fmt"
"regexp"
"strings"
"time"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/cveapi"
@@ -27,33 +25,24 @@ func newBsd(c config.ServerInfo) *bsd {
//https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/freebsd.rb
func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) {
bsd = newBsd(c)
//set sudo option flag
c.SudoOpt = config.SudoOption{ExecBySudo: true}
bsd.setServerInfo(c)
c.Family = "FreeBSD"
if r := sshExec(c, "uname", noSudo); r.isSuccess() {
if strings.Contains(r.Stdout, "FreeBSD") == true {
if b := sshExec(c, "uname -r", noSudo); b.isSuccess() {
bsd.setDistributionInfo("FreeBSD", b.Stdout)
} else {
return false, bsd
bsd.setDistributionInfo("FreeBSD", strings.TrimSpace(b.Stdout))
bsd.setServerInfo(c)
return true, bsd
}
return true, bsd
} else {
return false, bsd
}
}
return
return false, bsd
}
func (o *bsd) install() error {
//pkg upgrade
cmd := "pkg upgrade"
if r := o.ssh(cmd, sudo); !r.isSuccess() {
msg := fmt.Sprintf("Failed to %s. status: %d, stdout: %s, stderr: %s",
cmd, r.ExitStatus, r.Stdout, r.Stderr)
o.log.Errorf(msg)
return fmt.Errorf(msg)
}
return nil
}
func (o *bsd) checkRequiredPackagesInstalled() error {
return nil
}
@@ -65,8 +54,9 @@ func (o *bsd) scanPackages() error {
return err
}
o.setPackages(packs)
var unsecurePacks []CvePacksInfo
if unsecurePacks, err = o.scanUnsecurePackages(packs); err != nil {
if unsecurePacks, err = o.scanUnsecurePackages(); err != nil {
o.log.Errorf("Failed to scan vulnerable packages")
return err
}
@@ -74,181 +64,163 @@ func (o *bsd) scanPackages() error {
return nil
}
func (o *bsd) scanInstalledPackages() (packs []models.PackageInfo, err error) {
//pkg query is a FreeBSD to provide info on a certain package : %n=name %v=version: formatting for string split later
r := o.ssh("pkg query '%n*%v'", noSudo)
if !r.isSuccess() {
return packs, fmt.Errorf(
"Failed to scan packages. status: %d, stdout:%s, Stderr: %s",
r.ExitStatus, r.Stdout, r.Stderr)
}
//same format as debain.go
lines := strings.Split(r.Stdout, "\n")
for _, line := range lines {
//for every \n
if trimmed := strings.TrimSpace(line); len(trimmed) != 0 {
name, version, err := o.parseScanedPackagesLine(trimmed)
if err != nil {
return nil, fmt.Errorf("FreeBSD: Failed to parse package")
}
packs = append(packs, models.PackageInfo{
Name: name,
Version: version,
})
}
}
return
}
func (o *bsd) parseScanedPackagesLine(line string) (name, version string, err error) {
name = strings.Split(line, "*")[0]
if len(strings.Split(line, "*")) == 2 {
version = strings.Split(line, "*")[1]
}
return name, version, nil
}
func (o *bsd) checkRequiredPackagesInstalled() error {
return nil
}
func (o *bsd) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInfo, error) {
cmd := util.PrependProxyEnv("pkg version -l '>'")
func (o *bsd) scanInstalledPackages() ([]models.PackageInfo, error) {
cmd := util.PrependProxyEnv("pkg version -v")
r := o.ssh(cmd, noSudo)
if !r.isSuccess() {
return nil, nil
return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s",
cmd, r.ExitStatus, r.Stdout, r.Stderr)
}
match := regexp.MustCompile("(.)+[^[-](\\d)]")
upgradablePackNames := match.FindAllString(r.Stdout, -1)
// Convert package name to PackageInfo struct
var unsecurePacks []models.PackageInfo
var err error
for _, name := range upgradablePackNames {
for _, pack := range packs {
if pack.Name == name {
unsecurePacks = append(unsecurePacks, pack)
break
}
}
}
/* unsecurePacks, err = o.fillCanidateVersion(unsecurePacks)
if err != nil {
return nil, fmt.Errorf("Failed to fill canidate versions. err: %s", err)
}
*/
// Collect CVE information of upgradable packages
cvePacksInfos, err := o.scanPackageCveInfos(unsecurePacks)
if err != nil {
return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err)
}
return cvePacksInfos, nil
return o.parsePkgVersion(r.Stdout), nil
}
func (o *bsd) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePacksList CvePacksList, err error) {
func (o *bsd) scanUnsecurePackages() (cvePacksList []CvePacksInfo, err error) {
cmd := util.PrependProxyEnv("pkg audit -F -f /tmp/vuln.db -r")
r := o.ssh(cmd, noSudo)
if !r.isSuccess(0, 1) {
return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s",
cmd, r.ExitStatus, r.Stdout, r.Stderr)
}
if r.ExitStatus == 0 {
// no vulnerabilities
return []CvePacksInfo{}, nil
}
// { CVE ID: [packageInfo] }
cvePackages := make(map[string][]models.PackageInfo)
type strarray []string
resChan := make(chan struct {
models.PackageInfo
strarray
}, len(unsecurePacks))
errChan := make(chan error, len(unsecurePacks))
reqChan := make(chan models.PackageInfo, len(unsecurePacks))
defer close(resChan)
defer close(errChan)
defer close(reqChan)
go func() {
for _, pack := range unsecurePacks {
reqChan <- pack
var packAdtRslt []pkgAuditResult
blocks := o.splitIntoBlocks(r.Stdout)
for _, b := range blocks {
name, cveIDs, vulnID := o.parseBlock(b)
if len(cveIDs) == 0 {
continue
}
}()
pack, found := o.Packages.FindByName(name)
if !found {
return nil, fmt.Errorf("Vulnerable package: %s is not found", name)
}
packAdtRslt = append(packAdtRslt, pkgAuditResult{
pack: pack,
vulnIDCveIDs: vulnIDCveIDs{
vulnID: vulnID,
cveIDs: cveIDs,
},
})
}
timeout := time.After(30 * 60 * time.Second)
concurrency := 10
tasks := util.GenWorkers(concurrency)
for range unsecurePacks {
tasks <- func() {
select {
case pack := <-reqChan:
func(p models.PackageInfo) {
if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
errChan <- err
} else {
resChan <- struct {
models.PackageInfo
strarray
}{p, cveIDs}
}
}(pack)
}
// { CVE ID: []pkgAuditResult }
cveIDAdtMap := make(map[string][]pkgAuditResult)
for _, p := range packAdtRslt {
for _, cid := range p.vulnIDCveIDs.cveIDs {
cveIDAdtMap[cid] = append(cveIDAdtMap[cid], p)
}
}
errs := []error{}
for i := 0; i < len(unsecurePacks); i++ {
o.log.Info(unsecurePacks[i])
select {
case pair := <-resChan:
pack := pair.PackageInfo
cveIDs := pair.strarray
for _, cveID := range cveIDs {
cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack)
}
o.log.Infof("(%d/%d) Scanned %s-%s : %s",
i+1, len(unsecurePacks), pair.Name, pair.PackageInfo.Version, cveIDs)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return nil, fmt.Errorf("Timeout scanPackageCveIDs")
}
}
if 0 < len(errs) {
return nil, fmt.Errorf("%v", errs)
}
var cveIDs []string
for k := range cvePackages {
cveIDs := []string{}
for k := range cveIDAdtMap {
cveIDs = append(cveIDs, k)
}
o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs)
o.log.Info("Fetching CVE details...")
cveDetails, err := cveapi.CveClient.FetchCveDetails(cveIDs)
if err != nil {
return nil, err
}
o.log.Info("Done")
for _, detail := range cveDetails {
for _, d := range cveDetails {
packs := []models.PackageInfo{}
for _, r := range cveIDAdtMap[d.CveID] {
packs = append(packs, r.pack)
}
disAdvs := []models.DistroAdvisory{}
for _, r := range cveIDAdtMap[d.CveID] {
disAdvs = append(disAdvs, models.DistroAdvisory{
AdvisoryID: r.vulnIDCveIDs.vulnID,
})
}
cvePacksList = append(cvePacksList, CvePacksInfo{
CveID: detail.CveID,
CveDetail: detail,
Packs: cvePackages[detail.CveID],
// CvssScore: cinfo.CvssScore(conf.Lang),
CveID: d.CveID,
CveDetail: d,
Packs: packs,
DistroAdvisories: disAdvs,
})
}
return
}
func (o *bsd) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) {
cmd := fmt.Sprintf("pkg audit -F %s | grep CVE-\\w", pack.Name)
cmd = util.PrependProxyEnv(cmd)
func (o *bsd) parsePkgVersion(stdout string) (packs []models.PackageInfo) {
lines := strings.Split(stdout, "\n")
for _, l := range lines {
fields := strings.Fields(l)
if len(fields) < 2 {
continue
}
r := o.ssh(cmd, noSudo)
if !r.isSuccess() {
o.log.Warnf("Failed to %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr)
return nil, nil
packVer := fields[0]
splitted := strings.Split(packVer, "-")
ver := splitted[len(splitted)-1]
name := strings.Join(splitted[:len(splitted)-1], "-")
switch fields[1] {
case "?", "=":
packs = append(packs, models.PackageInfo{
Name: name,
Version: ver,
})
case "<":
candidate := strings.TrimSuffix(fields[6], ")")
packs = append(packs, models.PackageInfo{
Name: name,
Version: ver,
NewVersion: candidate,
})
}
}
match := regexp.MustCompile("(CVE-\\d{4}-\\d{4})")
return match.FindAllString(r.Stdout, -1), nil
return
}
type vulnIDCveIDs struct {
vulnID string
cveIDs []string
}
type pkgAuditResult struct {
pack models.PackageInfo
vulnIDCveIDs vulnIDCveIDs
}
func (o *bsd) splitIntoBlocks(stdout string) (blocks []string) {
lines := strings.Split(stdout, "\n")
block := []string{}
for _, l := range lines {
if len(strings.TrimSpace(l)) == 0 {
if 0 < len(block) {
blocks = append(blocks, strings.Join(block, "\n"))
block = []string{}
}
continue
}
block = append(block, strings.TrimSpace(l))
}
if 0 < len(block) {
blocks = append(blocks, strings.Join(block, "\n"))
}
return
}
func (o *bsd) parseBlock(block string) (packName string, cveIDs []string, vulnID string) {
lines := strings.Split(block, "\n")
for _, l := range lines {
if strings.HasSuffix(l, " is vulnerable:") {
packVer := strings.Fields(l)[0]
splitted := strings.Split(packVer, "-")
packName = strings.Join(splitted[:len(splitted)-1], "-")
} else if strings.HasPrefix(l, "CVE:") {
cveIDs = append(cveIDs, strings.Fields(l)[1])
} else if strings.HasPrefix(l, "WWW:") {
splitted := strings.Split(l, "/")
vulnID = strings.TrimSuffix(splitted[len(splitted)-1], ".html")
}
}
return
}

155
scan/freebsd_test.go Normal file
View File

@@ -0,0 +1,155 @@
package scan
import (
"reflect"
"testing"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/k0kubun/pp"
)
func TestParsePkgVersion(t *testing.T) {
var tests = []struct {
in string
expected []models.PackageInfo
}{
{
`Updating FreeBSD repository catalogue...
FreeBSD repository is up-to-date.
All repositories are up-to-date.
bash-4.2.45 < needs updating (remote has 4.3.42_1)
gettext-0.18.3.1 < needs updating (remote has 0.19.7)
tcl84-8.4.20_2,1 = up-to-date with remote
teTeX-base-3.0_25 ? orphaned: print/teTeX-base`,
[]models.PackageInfo{
{
Name: "bash",
Version: "4.2.45",
NewVersion: "4.3.42_1",
},
{
Name: "gettext",
Version: "0.18.3.1",
NewVersion: "0.19.7",
},
{
Name: "tcl84",
Version: "8.4.20_2,1",
},
{
Name: "teTeX-base",
Version: "3.0_25",
},
},
},
}
d := newBsd(config.ServerInfo{})
for _, tt := range tests {
actual := d.parsePkgVersion(tt.in)
if !reflect.DeepEqual(tt.expected, actual) {
e := pp.Sprintf("%v", tt.expected)
a := pp.Sprintf("%v", actual)
t.Errorf("expected %s, actual %s", e, a)
}
}
}
func TestSplitIntoBlocks(t *testing.T) {
var tests = []struct {
in string
expected []string
}{
{
`
block1
block2
block2
block2
block3
block3`,
[]string{
`block1`,
"block2\nblock2\nblock2",
"block3\nblock3",
},
},
}
d := newBsd(config.ServerInfo{})
for _, tt := range tests {
actual := d.splitIntoBlocks(tt.in)
if !reflect.DeepEqual(tt.expected, actual) {
e := pp.Sprintf("%v", tt.expected)
a := pp.Sprintf("%v", actual)
t.Errorf("expected %s, actual %s", e, a)
}
}
}
func TestParseBlock(t *testing.T) {
var tests = []struct {
in string
name string
cveIDs []string
vulnID string
}{
{
in: `vulnxml file up-to-date
bind96-9.6.3.2.ESV.R10_2 is vulnerable:
bind -- denial of service vulnerability
CVE: CVE-2014-0591
WWW: https://vuxml.FreeBSD.org/freebsd/cb252f01-7c43-11e3-b0a6-005056a37f68.html`,
name: "bind96",
cveIDs: []string{"CVE-2014-0591"},
vulnID: "cb252f01-7c43-11e3-b0a6-005056a37f68",
},
{
in: `bind96-9.6.3.2.ESV.R10_2 is vulnerable:
bind -- denial of service vulnerability
CVE: CVE-2014-8680
CVE: CVE-2014-8500
WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html`,
name: "bind96",
cveIDs: []string{"CVE-2014-8680", "CVE-2014-8500"},
vulnID: "ab3e98d9-8175-11e4-907d-d050992ecde8",
},
{
in: `hoge-hoge-9.6.3.2.ESV.R10_2 is vulnerable:
bind -- denial of service vulnerability
CVE: CVE-2014-8680
CVE: CVE-2014-8500
WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html`,
name: "hoge-hoge",
cveIDs: []string{"CVE-2014-8680", "CVE-2014-8500"},
vulnID: "ab3e98d9-8175-11e4-907d-d050992ecde8",
},
{
in: `1 problem(s) in the installed packages found.`,
cveIDs: []string{},
vulnID: "",
},
}
d := newBsd(config.ServerInfo{})
for _, tt := range tests {
aName, aCveIDs, aVunlnID := d.parseBlock(tt.in)
if tt.name != aName {
t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID)
}
for i := range tt.cveIDs {
if tt.cveIDs[i] != aCveIDs[i] {
t.Errorf("expected cveID: %s, actual %s", tt.cveIDs[i], aCveIDs[i])
}
}
if tt.vulnID != aVunlnID {
t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID)
}
}
}

View File

@@ -64,10 +64,9 @@ type CvePacksList []CvePacksInfo
type CvePacksInfo struct {
CveID string
CveDetail cve.CveDetail
Packs []models.PackageInfo
DistroAdvisories []models.DistroAdvisory // for Aamazon, RHEL
Packs models.PackageInfoList
DistroAdvisories []models.DistroAdvisory // for Aamazon, RHEL, FreeBSD
CpeNames []string
// CvssScore float64
}
// FindByCveID find by CVEID

View File

@@ -134,11 +134,12 @@ func sshExec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (re
logger.Panicf("sudoOpt is invalid. SudoOpt: %v", c.SudoOpt)
}
}
// set pipefail option.
// http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another
cmd = fmt.Sprintf("set -o pipefail; %s", cmd)
logger.Debugf("Command: %s",
strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1))
if c.Family != "FreeBSD" {
// set pipefail option. Bash only
// http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another
cmd = fmt.Sprintf("set -o pipefail; %s", cmd)
}
if c.IsContainer() {
switch c.Container.Type {
@@ -147,6 +148,9 @@ func sshExec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (re
}
}
logger.Debugf("Command: %s",
strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1))
var client *ssh.Client
client, err = sshConnect(c)
defer client.Close()