Enable to detect vulnerabilities on FreeBSD

scan/debian.go

@@ -194,7 +194,7 @@ func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error)
 	lines := strings.Split(r.Stdout, "\n")
 	for _, line := range lines {
 		if trimmed := strings.TrimSpace(line); len(trimmed) != 0 {
-			name, version, err := o.parseScanedPackagesLine(trimmed)
+			name, version, err := o.parseScannedPackagesLine(trimmed)
 			if err != nil {
 				return nil, fmt.Errorf(
 					"Debian: Failed to parse package line: %s", line)
@@ -208,7 +208,7 @@ func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error)
 	return
 }
 
-func (o *debian) parseScanedPackagesLine(line string) (name, version string, err error) {
+func (o *debian) parseScannedPackagesLine(line string) (name, version string, err error) {
 	re, _ := regexp.Compile(`^([^\t']+)\t(.+)$`)
 	result := re.FindStringSubmatch(line)
 	if len(result) == 3 {

scan/debian_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/k0kubun/pp"
 )
 
-func TestParseScanedPackagesLineDebian(t *testing.T) {
+func TestParseScannedPackagesLineDebian(t *testing.T) {
 
 	var packagetests = []struct {
 		in      string
@@ -43,7 +43,7 @@ func TestParseScanedPackagesLineDebian(t *testing.T) {
 
 	d := newDebian(config.ServerInfo{})
 	for _, tt := range packagetests {
-		n, v, _ := d.parseScanedPackagesLine(tt.in)
+		n, v, _ := d.parseScannedPackagesLine(tt.in)
 		if n != tt.name {
 			t.Errorf("name: expected %s, actual %s", tt.name, n)
 		}

scan/freebsd.go

@@ -2,9 +2,7 @@ package scan
 
 import (
 	"fmt"
-	"regexp"
 	"strings"
-	"time"
 
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/cveapi"
@@ -27,33 +25,24 @@ func newBsd(c config.ServerInfo) *bsd {
 //https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/freebsd.rb
 func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) {
 	bsd = newBsd(c)
-	//set sudo option flag
-	c.SudoOpt = config.SudoOption{ExecBySudo: true}
-	bsd.setServerInfo(c)
-
+	c.Family = "FreeBSD"
 	if r := sshExec(c, "uname", noSudo); r.isSuccess() {
 		if strings.Contains(r.Stdout, "FreeBSD") == true {
 			if b := sshExec(c, "uname -r", noSudo); b.isSuccess() {
-				bsd.setDistributionInfo("FreeBSD", b.Stdout)
-			} else {
-				return false, bsd
+				bsd.setDistributionInfo("FreeBSD", strings.TrimSpace(b.Stdout))
+				bsd.setServerInfo(c)
+				return true, bsd
 			}
-			return true, bsd
-		} else {
-			return false, bsd
 		}
 	}
-	return
+	return false, bsd
 }
+
 func (o *bsd) install() error {
-	//pkg upgrade
-	cmd := "pkg upgrade"
-	if r := o.ssh(cmd, sudo); !r.isSuccess() {
-		msg := fmt.Sprintf("Failed to %s. status: %d, stdout: %s, stderr: %s",
-			cmd, r.ExitStatus, r.Stdout, r.Stderr)
-		o.log.Errorf(msg)
-		return fmt.Errorf(msg)
-	}
+	return nil
+}
+
+func (o *bsd) checkRequiredPackagesInstalled() error {
 	return nil
 }
 
@@ -65,8 +54,9 @@ func (o *bsd) scanPackages() error {
 		return err
 	}
 	o.setPackages(packs)
+
 	var unsecurePacks []CvePacksInfo
-	if unsecurePacks, err = o.scanUnsecurePackages(packs); err != nil {
+	if unsecurePacks, err = o.scanUnsecurePackages(); err != nil {
 		o.log.Errorf("Failed to scan vulnerable packages")
 		return err
 	}
@@ -74,181 +64,163 @@ func (o *bsd) scanPackages() error {
 	return nil
 }
 
-func (o *bsd) scanInstalledPackages() (packs []models.PackageInfo, err error) {
-	//pkg query is a FreeBSD to provide info on a certain package : %n=name %v=version: formatting for string split later
-	r := o.ssh("pkg query '%n*%v'", noSudo)
-	if !r.isSuccess() {
-		return packs, fmt.Errorf(
-			"Failed to scan packages. status: %d, stdout:%s, Stderr: %s",
-			r.ExitStatus, r.Stdout, r.Stderr)
-	}
-	//same format as debain.go
-	lines := strings.Split(r.Stdout, "\n")
-	for _, line := range lines {
-		//for every \n
-		if trimmed := strings.TrimSpace(line); len(trimmed) != 0 {
-			name, version, err := o.parseScanedPackagesLine(trimmed)
-			if err != nil {
-				return nil, fmt.Errorf("FreeBSD: Failed to parse package")
-			}
-			packs = append(packs, models.PackageInfo{
-				Name:    name,
-				Version: version,
-			})
-		}
-	}
-	return
-}
-
-func (o *bsd) parseScanedPackagesLine(line string) (name, version string, err error) {
-	name = strings.Split(line, "*")[0]
-	if len(strings.Split(line, "*")) == 2 {
-
-		version = strings.Split(line, "*")[1]
-	}
-	return name, version, nil
-}
-
-func (o *bsd) checkRequiredPackagesInstalled() error {
-	return nil
-}
-
-func (o *bsd) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInfo, error) {
-	cmd := util.PrependProxyEnv("pkg version -l '>'")
+func (o *bsd) scanInstalledPackages() ([]models.PackageInfo, error) {
+	cmd := util.PrependProxyEnv("pkg version -v")
 	r := o.ssh(cmd, noSudo)
 	if !r.isSuccess() {
-		return nil, nil
+		return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s",
+			cmd, r.ExitStatus, r.Stdout, r.Stderr)
 	}
-	match := regexp.MustCompile("(.)+[^[-](\\d)]")
-	upgradablePackNames := match.FindAllString(r.Stdout, -1)
-
-	// Convert package name to PackageInfo struct
-	var unsecurePacks []models.PackageInfo
-	var err error
-	for _, name := range upgradablePackNames {
-		for _, pack := range packs {
-			if pack.Name == name {
-				unsecurePacks = append(unsecurePacks, pack)
-				break
-			}
-		}
-	}
-	/* unsecurePacks, err = o.fillCanidateVersion(unsecurePacks)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to fill canidate versions. err: %s", err)
-	}
-	*/
-
-	// Collect CVE information of upgradable packages
-	cvePacksInfos, err := o.scanPackageCveInfos(unsecurePacks)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err)
-	}
-
-	return cvePacksInfos, nil
+	return o.parsePkgVersion(r.Stdout), nil
 }
 
-func (o *bsd) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePacksList CvePacksList, err error) {
+func (o *bsd) scanUnsecurePackages() (cvePacksList []CvePacksInfo, err error) {
+	cmd := util.PrependProxyEnv("pkg audit -F -f /tmp/vuln.db -r")
+	r := o.ssh(cmd, noSudo)
+	if !r.isSuccess(0, 1) {
+		return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s",
+			cmd, r.ExitStatus, r.Stdout, r.Stderr)
+	}
+	if r.ExitStatus == 0 {
+		// no vulnerabilities
+		return []CvePacksInfo{}, nil
+	}
 
-	// { CVE ID: [packageInfo] }
-	cvePackages := make(map[string][]models.PackageInfo)
-
-	type strarray []string
-	resChan := make(chan struct {
-		models.PackageInfo
-		strarray
-	}, len(unsecurePacks))
-	errChan := make(chan error, len(unsecurePacks))
-	reqChan := make(chan models.PackageInfo, len(unsecurePacks))
-	defer close(resChan)
-	defer close(errChan)
-	defer close(reqChan)
-
-	go func() {
-		for _, pack := range unsecurePacks {
-			reqChan <- pack
+	var packAdtRslt []pkgAuditResult
+	blocks := o.splitIntoBlocks(r.Stdout)
+	for _, b := range blocks {
+		name, cveIDs, vulnID := o.parseBlock(b)
+		if len(cveIDs) == 0 {
+			continue
 		}
-	}()
+		pack, found := o.Packages.FindByName(name)
+		if !found {
+			return nil, fmt.Errorf("Vulnerable package: %s is not found", name)
+		}
+		packAdtRslt = append(packAdtRslt, pkgAuditResult{
+			pack: pack,
+			vulnIDCveIDs: vulnIDCveIDs{
+				vulnID: vulnID,
+				cveIDs: cveIDs,
+			},
+		})
+	}
 
-	timeout := time.After(30 * 60 * time.Second)
-
-	concurrency := 10
-	tasks := util.GenWorkers(concurrency)
-	for range unsecurePacks {
-		tasks <- func() {
-			select {
-			case pack := <-reqChan:
-				func(p models.PackageInfo) {
-					if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
-						errChan <- err
-					} else {
-						resChan <- struct {
-							models.PackageInfo
-							strarray
-						}{p, cveIDs}
-					}
-				}(pack)
-			}
+	// { CVE ID: []pkgAuditResult }
+	cveIDAdtMap := make(map[string][]pkgAuditResult)
+	for _, p := range packAdtRslt {
+		for _, cid := range p.vulnIDCveIDs.cveIDs {
+			cveIDAdtMap[cid] = append(cveIDAdtMap[cid], p)
 		}
 	}
 
-	errs := []error{}
-	for i := 0; i < len(unsecurePacks); i++ {
-		o.log.Info(unsecurePacks[i])
-		select {
-		case pair := <-resChan:
-			pack := pair.PackageInfo
-			cveIDs := pair.strarray
-			for _, cveID := range cveIDs {
-				cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack)
-			}
-			o.log.Infof("(%d/%d) Scanned %s-%s : %s",
-				i+1, len(unsecurePacks), pair.Name, pair.PackageInfo.Version, cveIDs)
-		case err := <-errChan:
-			errs = append(errs, err)
-		case <-timeout:
-			return nil, fmt.Errorf("Timeout scanPackageCveIDs")
-		}
-	}
-
-	if 0 < len(errs) {
-		return nil, fmt.Errorf("%v", errs)
-	}
-
-	var cveIDs []string
-	for k := range cvePackages {
+	cveIDs := []string{}
+	for k := range cveIDAdtMap {
 		cveIDs = append(cveIDs, k)
 	}
 
-	o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs)
-
-	o.log.Info("Fetching CVE details...")
 	cveDetails, err := cveapi.CveClient.FetchCveDetails(cveIDs)
 	if err != nil {
 		return nil, err
 	}
 	o.log.Info("Done")
 
-	for _, detail := range cveDetails {
+	for _, d := range cveDetails {
+		packs := []models.PackageInfo{}
+		for _, r := range cveIDAdtMap[d.CveID] {
+			packs = append(packs, r.pack)
+		}
+
+		disAdvs := []models.DistroAdvisory{}
+		for _, r := range cveIDAdtMap[d.CveID] {
+			disAdvs = append(disAdvs, models.DistroAdvisory{
+				AdvisoryID: r.vulnIDCveIDs.vulnID,
+			})
+		}
+
 		cvePacksList = append(cvePacksList, CvePacksInfo{
-			CveID:     detail.CveID,
-			CveDetail: detail,
-			Packs:     cvePackages[detail.CveID],
-			//  CvssScore: cinfo.CvssScore(conf.Lang),
+			CveID:            d.CveID,
+			CveDetail:        d,
+			Packs:            packs,
+			DistroAdvisories: disAdvs,
 		})
 	}
 	return
 }
 
-func (o *bsd) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) {
-	cmd := fmt.Sprintf("pkg audit -F %s | grep CVE-\\w", pack.Name)
-	cmd = util.PrependProxyEnv(cmd)
+func (o *bsd) parsePkgVersion(stdout string) (packs []models.PackageInfo) {
+	lines := strings.Split(stdout, "\n")
+	for _, l := range lines {
+		fields := strings.Fields(l)
+		if len(fields) < 2 {
+			continue
+		}
 
-	r := o.ssh(cmd, noSudo)
-	if !r.isSuccess() {
-		o.log.Warnf("Failed to %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr)
-		return nil, nil
+		packVer := fields[0]
+		splitted := strings.Split(packVer, "-")
+		ver := splitted[len(splitted)-1]
+		name := strings.Join(splitted[:len(splitted)-1], "-")
+
+		switch fields[1] {
+		case "?", "=":
+			packs = append(packs, models.PackageInfo{
+				Name:    name,
+				Version: ver,
+			})
+		case "<":
+			candidate := strings.TrimSuffix(fields[6], ")")
+			packs = append(packs, models.PackageInfo{
+				Name:       name,
+				Version:    ver,
+				NewVersion: candidate,
+			})
+		}
 	}
-	match := regexp.MustCompile("(CVE-\\d{4}-\\d{4})")
-	return match.FindAllString(r.Stdout, -1), nil
+	return
+}
+
+type vulnIDCveIDs struct {
+	vulnID string
+	cveIDs []string
+}
+
+type pkgAuditResult struct {
+	pack         models.PackageInfo
+	vulnIDCveIDs vulnIDCveIDs
+}
+
+func (o *bsd) splitIntoBlocks(stdout string) (blocks []string) {
+	lines := strings.Split(stdout, "\n")
+	block := []string{}
+	for _, l := range lines {
+		if len(strings.TrimSpace(l)) == 0 {
+			if 0 < len(block) {
+				blocks = append(blocks, strings.Join(block, "\n"))
+				block = []string{}
+			}
+			continue
+		}
+		block = append(block, strings.TrimSpace(l))
+	}
+	if 0 < len(block) {
+		blocks = append(blocks, strings.Join(block, "\n"))
+	}
+	return
+}
+
+func (o *bsd) parseBlock(block string) (packName string, cveIDs []string, vulnID string) {
+	lines := strings.Split(block, "\n")
+	for _, l := range lines {
+		if strings.HasSuffix(l, " is vulnerable:") {
+			packVer := strings.Fields(l)[0]
+			splitted := strings.Split(packVer, "-")
+			packName = strings.Join(splitted[:len(splitted)-1], "-")
+		} else if strings.HasPrefix(l, "CVE:") {
+			cveIDs = append(cveIDs, strings.Fields(l)[1])
+		} else if strings.HasPrefix(l, "WWW:") {
+			splitted := strings.Split(l, "/")
+			vulnID = strings.TrimSuffix(splitted[len(splitted)-1], ".html")
+		}
+	}
+	return
 }

scan/freebsd_test.go

@@ -0,0 +1,155 @@
+package scan
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
+	"github.com/k0kubun/pp"
+)
+
+func TestParsePkgVersion(t *testing.T) {
+	var tests = []struct {
+		in       string
+		expected []models.PackageInfo
+	}{
+		{
+			`Updating FreeBSD repository catalogue...
+FreeBSD repository is up-to-date.
+All repositories are up-to-date.
+bash-4.2.45                        <   needs updating (remote has 4.3.42_1)
+gettext-0.18.3.1                   <   needs updating (remote has 0.19.7)
+tcl84-8.4.20_2,1                   =   up-to-date with remote
+teTeX-base-3.0_25                  ?   orphaned: print/teTeX-base`,
+
+			[]models.PackageInfo{
+				{
+					Name:       "bash",
+					Version:    "4.2.45",
+					NewVersion: "4.3.42_1",
+				},
+				{
+					Name:       "gettext",
+					Version:    "0.18.3.1",
+					NewVersion: "0.19.7",
+				},
+				{
+					Name:    "tcl84",
+					Version: "8.4.20_2,1",
+				},
+				{
+					Name:    "teTeX-base",
+					Version: "3.0_25",
+				},
+			},
+		},
+	}
+
+	d := newBsd(config.ServerInfo{})
+	for _, tt := range tests {
+		actual := d.parsePkgVersion(tt.in)
+		if !reflect.DeepEqual(tt.expected, actual) {
+			e := pp.Sprintf("%v", tt.expected)
+			a := pp.Sprintf("%v", actual)
+			t.Errorf("expected %s, actual %s", e, a)
+		}
+	}
+}
+
+func TestSplitIntoBlocks(t *testing.T) {
+	var tests = []struct {
+		in       string
+		expected []string
+	}{
+		{
+			`
+block1
+
+block2
+block2
+block2
+
+block3
+block3`,
+			[]string{
+				`block1`,
+				"block2\nblock2\nblock2",
+				"block3\nblock3",
+			},
+		},
+	}
+
+	d := newBsd(config.ServerInfo{})
+	for _, tt := range tests {
+		actual := d.splitIntoBlocks(tt.in)
+		if !reflect.DeepEqual(tt.expected, actual) {
+			e := pp.Sprintf("%v", tt.expected)
+			a := pp.Sprintf("%v", actual)
+			t.Errorf("expected %s, actual %s", e, a)
+		}
+	}
+
+}
+
+func TestParseBlock(t *testing.T) {
+	var tests = []struct {
+		in     string
+		name   string
+		cveIDs []string
+		vulnID string
+	}{
+		{
+
+			in: `vulnxml file up-to-date
+bind96-9.6.3.2.ESV.R10_2 is vulnerable:
+bind -- denial of service vulnerability
+CVE: CVE-2014-0591
+WWW: https://vuxml.FreeBSD.org/freebsd/cb252f01-7c43-11e3-b0a6-005056a37f68.html`,
+			name:   "bind96",
+			cveIDs: []string{"CVE-2014-0591"},
+			vulnID: "cb252f01-7c43-11e3-b0a6-005056a37f68",
+		},
+		{
+			in: `bind96-9.6.3.2.ESV.R10_2 is vulnerable:
+bind -- denial of service vulnerability
+CVE: CVE-2014-8680
+CVE: CVE-2014-8500
+WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html`,
+			name:   "bind96",
+			cveIDs: []string{"CVE-2014-8680", "CVE-2014-8500"},
+			vulnID: "ab3e98d9-8175-11e4-907d-d050992ecde8",
+		},
+		{
+			in: `hoge-hoge-9.6.3.2.ESV.R10_2 is vulnerable:
+bind -- denial of service vulnerability
+CVE: CVE-2014-8680
+CVE: CVE-2014-8500
+WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html`,
+			name:   "hoge-hoge",
+			cveIDs: []string{"CVE-2014-8680", "CVE-2014-8500"},
+			vulnID: "ab3e98d9-8175-11e4-907d-d050992ecde8",
+		},
+		{
+			in:     `1 problem(s) in the installed packages found.`,
+			cveIDs: []string{},
+			vulnID: "",
+		},
+	}
+
+	d := newBsd(config.ServerInfo{})
+	for _, tt := range tests {
+		aName, aCveIDs, aVunlnID := d.parseBlock(tt.in)
+		if tt.name != aName {
+			t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID)
+		}
+		for i := range tt.cveIDs {
+			if tt.cveIDs[i] != aCveIDs[i] {
+				t.Errorf("expected cveID: %s, actual %s", tt.cveIDs[i], aCveIDs[i])
+			}
+		}
+		if tt.vulnID != aVunlnID {
+			t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID)
+		}
+	}
+}

scan/serverapi.go

@@ -64,10 +64,9 @@ type CvePacksList []CvePacksInfo
 type CvePacksInfo struct {
 	CveID            string
 	CveDetail        cve.CveDetail
-	Packs            []models.PackageInfo
-	DistroAdvisories []models.DistroAdvisory // for Aamazon, RHEL
+	Packs            models.PackageInfoList
+	DistroAdvisories []models.DistroAdvisory // for Aamazon, RHEL, FreeBSD
 	CpeNames         []string
-	//  CvssScore float64
 }
 
 // FindByCveID find by CVEID

scan/sshutil.go

@@ -134,11 +134,12 @@ func sshExec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (re
 			logger.Panicf("sudoOpt is invalid. SudoOpt: %v", c.SudoOpt)
 		}
 	}
-	// set pipefail option.
-	// http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another
-	cmd = fmt.Sprintf("set -o pipefail; %s", cmd)
-	logger.Debugf("Command: %s",
-		strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1))
+
+	if c.Family != "FreeBSD" {
+		// set pipefail option. Bash only
+		// http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another
+		cmd = fmt.Sprintf("set -o pipefail; %s", cmd)
+	}
 
 	if c.IsContainer() {
 		switch c.Container.Type {
@@ -147,6 +148,9 @@ func sshExec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (re
 		}
 	}
 
+	logger.Debugf("Command: %s",
+		strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1))
+
 	var client *ssh.Client
 	client, err = sshConnect(c)
 	defer client.Close()