From 17181405e3f179999fc77becbb32eef9f5669bca Mon Sep 17 00:00:00 2001 From: kota kanbe Date: Mon, 13 Jun 2016 22:41:48 +0900 Subject: [PATCH] Enable to detect vulnerabilities on FreeBSD --- config/config.go | 3 +- models/models.go | 2 +- report/util.go | 9 ++ report/writer.go | 2 + scan/debian.go | 4 +- scan/debian_test.go | 4 +- scan/freebsd.go | 314 ++++++++++++++++++++----------------------- scan/freebsd_test.go | 155 +++++++++++++++++++++ scan/serverapi.go | 5 +- scan/sshutil.go | 14 +- 10 files changed, 327 insertions(+), 185 deletions(-) create mode 100644 scan/freebsd_test.go diff --git a/config/config.go b/config/config.go index 292e45c5..7004a75b 100644 --- a/config/config.go +++ b/config/config.go @@ -221,10 +221,11 @@ type ServerInfo struct { // Container Names or IDs Containers []string - // userd internal + // used internal LogMsgAnsiColor string // DebugLog Color SudoOpt SudoOption Container Container + Family string } // IsContainer returns whether this ServerInfo is about container diff --git a/models/models.go b/models/models.go index e80e4462..24c7cbb3 100644 --- a/models/models.go +++ b/models/models.go @@ -305,7 +305,7 @@ func (p PackageInfo) ToStringNewVersion() string { return str } -// DistroAdvisory has Amazon Linux AMI Security Advisory information. +// DistroAdvisory has Amazon Linux, RHEL, FreeBSD Security Advisory information. type DistroAdvisory struct { gorm.Model `json:"-"` CveInfoID uint `json:"-"` diff --git a/report/util.go b/report/util.go index 11d6f4aa..4875c92d 100644 --- a/report/util.go +++ b/report/util.go @@ -341,6 +341,15 @@ func distroLinks(cveInfo models.CveInfo, osFamily string) []distroLink { }, // TODO Debian dsa } + case "FreeBSD": + links := []distroLink{} + for _, advisory := range cveInfo.DistroAdvisories { + links = append(links, distroLink{ + "FreeBSD-VuXML", + fmt.Sprintf(freeBSDVuXMLBaseURL, advisory.AdvisoryID), + }) + } + return links default: return []distroLink{} } diff --git a/report/writer.go b/report/writer.go index 6afedd1c..272b1956 100644 --- a/report/writer.go +++ b/report/writer.go @@ -31,6 +31,8 @@ const ( ubuntuSecurityBaseURL = "http://people.ubuntu.com/~ubuntu-security/cve" debianTrackerBaseURL = "https://security-tracker.debian.org/tracker" + + freeBSDVuXMLBaseURL = "https://vuxml.freebsd.org/freebsd/%s.html" ) // ResultWriter Interface diff --git a/scan/debian.go b/scan/debian.go index e02c75a9..7b4b8244 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -194,7 +194,7 @@ func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error) lines := strings.Split(r.Stdout, "\n") for _, line := range lines { if trimmed := strings.TrimSpace(line); len(trimmed) != 0 { - name, version, err := o.parseScanedPackagesLine(trimmed) + name, version, err := o.parseScannedPackagesLine(trimmed) if err != nil { return nil, fmt.Errorf( "Debian: Failed to parse package line: %s", line) @@ -208,7 +208,7 @@ func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error) return } -func (o *debian) parseScanedPackagesLine(line string) (name, version string, err error) { +func (o *debian) parseScannedPackagesLine(line string) (name, version string, err error) { re, _ := regexp.Compile(`^([^\t']+)\t(.+)$`) result := re.FindStringSubmatch(line) if len(result) == 3 { diff --git a/scan/debian_test.go b/scan/debian_test.go index 98d8f8b0..40014379 100644 --- a/scan/debian_test.go +++ b/scan/debian_test.go @@ -25,7 +25,7 @@ import ( "github.com/k0kubun/pp" ) -func TestParseScanedPackagesLineDebian(t *testing.T) { +func TestParseScannedPackagesLineDebian(t *testing.T) { var packagetests = []struct { in string @@ -43,7 +43,7 @@ func TestParseScanedPackagesLineDebian(t *testing.T) { d := newDebian(config.ServerInfo{}) for _, tt := range packagetests { - n, v, _ := d.parseScanedPackagesLine(tt.in) + n, v, _ := d.parseScannedPackagesLine(tt.in) if n != tt.name { t.Errorf("name: expected %s, actual %s", tt.name, n) } diff --git a/scan/freebsd.go b/scan/freebsd.go index 2cb003a7..00dce6fc 100644 --- a/scan/freebsd.go +++ b/scan/freebsd.go @@ -2,9 +2,7 @@ package scan import ( "fmt" - "regexp" "strings" - "time" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/cveapi" @@ -27,33 +25,24 @@ func newBsd(c config.ServerInfo) *bsd { //https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/freebsd.rb func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) { bsd = newBsd(c) - //set sudo option flag - c.SudoOpt = config.SudoOption{ExecBySudo: true} - bsd.setServerInfo(c) - + c.Family = "FreeBSD" if r := sshExec(c, "uname", noSudo); r.isSuccess() { if strings.Contains(r.Stdout, "FreeBSD") == true { if b := sshExec(c, "uname -r", noSudo); b.isSuccess() { - bsd.setDistributionInfo("FreeBSD", b.Stdout) - } else { - return false, bsd + bsd.setDistributionInfo("FreeBSD", strings.TrimSpace(b.Stdout)) + bsd.setServerInfo(c) + return true, bsd } - return true, bsd - } else { - return false, bsd } } - return + return false, bsd } + func (o *bsd) install() error { - //pkg upgrade - cmd := "pkg upgrade" - if r := o.ssh(cmd, sudo); !r.isSuccess() { - msg := fmt.Sprintf("Failed to %s. status: %d, stdout: %s, stderr: %s", - cmd, r.ExitStatus, r.Stdout, r.Stderr) - o.log.Errorf(msg) - return fmt.Errorf(msg) - } + return nil +} + +func (o *bsd) checkRequiredPackagesInstalled() error { return nil } @@ -65,8 +54,9 @@ func (o *bsd) scanPackages() error { return err } o.setPackages(packs) + var unsecurePacks []CvePacksInfo - if unsecurePacks, err = o.scanUnsecurePackages(packs); err != nil { + if unsecurePacks, err = o.scanUnsecurePackages(); err != nil { o.log.Errorf("Failed to scan vulnerable packages") return err } @@ -74,181 +64,163 @@ func (o *bsd) scanPackages() error { return nil } -func (o *bsd) scanInstalledPackages() (packs []models.PackageInfo, err error) { - //pkg query is a FreeBSD to provide info on a certain package : %n=name %v=version: formatting for string split later - r := o.ssh("pkg query '%n*%v'", noSudo) - if !r.isSuccess() { - return packs, fmt.Errorf( - "Failed to scan packages. status: %d, stdout:%s, Stderr: %s", - r.ExitStatus, r.Stdout, r.Stderr) - } - //same format as debain.go - lines := strings.Split(r.Stdout, "\n") - for _, line := range lines { - //for every \n - if trimmed := strings.TrimSpace(line); len(trimmed) != 0 { - name, version, err := o.parseScanedPackagesLine(trimmed) - if err != nil { - return nil, fmt.Errorf("FreeBSD: Failed to parse package") - } - packs = append(packs, models.PackageInfo{ - Name: name, - Version: version, - }) - } - } - return -} - -func (o *bsd) parseScanedPackagesLine(line string) (name, version string, err error) { - name = strings.Split(line, "*")[0] - if len(strings.Split(line, "*")) == 2 { - - version = strings.Split(line, "*")[1] - } - return name, version, nil -} - -func (o *bsd) checkRequiredPackagesInstalled() error { - return nil -} - -func (o *bsd) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInfo, error) { - cmd := util.PrependProxyEnv("pkg version -l '>'") +func (o *bsd) scanInstalledPackages() ([]models.PackageInfo, error) { + cmd := util.PrependProxyEnv("pkg version -v") r := o.ssh(cmd, noSudo) if !r.isSuccess() { - return nil, nil + return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s", + cmd, r.ExitStatus, r.Stdout, r.Stderr) } - match := regexp.MustCompile("(.)+[^[-](\\d)]") - upgradablePackNames := match.FindAllString(r.Stdout, -1) - - // Convert package name to PackageInfo struct - var unsecurePacks []models.PackageInfo - var err error - for _, name := range upgradablePackNames { - for _, pack := range packs { - if pack.Name == name { - unsecurePacks = append(unsecurePacks, pack) - break - } - } - } - /* unsecurePacks, err = o.fillCanidateVersion(unsecurePacks) - if err != nil { - return nil, fmt.Errorf("Failed to fill canidate versions. err: %s", err) - } - */ - - // Collect CVE information of upgradable packages - cvePacksInfos, err := o.scanPackageCveInfos(unsecurePacks) - if err != nil { - return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err) - } - - return cvePacksInfos, nil + return o.parsePkgVersion(r.Stdout), nil } -func (o *bsd) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePacksList CvePacksList, err error) { +func (o *bsd) scanUnsecurePackages() (cvePacksList []CvePacksInfo, err error) { + cmd := util.PrependProxyEnv("pkg audit -F -f /tmp/vuln.db -r") + r := o.ssh(cmd, noSudo) + if !r.isSuccess(0, 1) { + return nil, fmt.Errorf("Failed to %s. status: %d, stdout:%s, Stderr: %s", + cmd, r.ExitStatus, r.Stdout, r.Stderr) + } + if r.ExitStatus == 0 { + // no vulnerabilities + return []CvePacksInfo{}, nil + } - // { CVE ID: [packageInfo] } - cvePackages := make(map[string][]models.PackageInfo) - - type strarray []string - resChan := make(chan struct { - models.PackageInfo - strarray - }, len(unsecurePacks)) - errChan := make(chan error, len(unsecurePacks)) - reqChan := make(chan models.PackageInfo, len(unsecurePacks)) - defer close(resChan) - defer close(errChan) - defer close(reqChan) - - go func() { - for _, pack := range unsecurePacks { - reqChan <- pack + var packAdtRslt []pkgAuditResult + blocks := o.splitIntoBlocks(r.Stdout) + for _, b := range blocks { + name, cveIDs, vulnID := o.parseBlock(b) + if len(cveIDs) == 0 { + continue } - }() + pack, found := o.Packages.FindByName(name) + if !found { + return nil, fmt.Errorf("Vulnerable package: %s is not found", name) + } + packAdtRslt = append(packAdtRslt, pkgAuditResult{ + pack: pack, + vulnIDCveIDs: vulnIDCveIDs{ + vulnID: vulnID, + cveIDs: cveIDs, + }, + }) + } - timeout := time.After(30 * 60 * time.Second) - - concurrency := 10 - tasks := util.GenWorkers(concurrency) - for range unsecurePacks { - tasks <- func() { - select { - case pack := <-reqChan: - func(p models.PackageInfo) { - if cveIDs, err := o.scanPackageCveIDs(p); err != nil { - errChan <- err - } else { - resChan <- struct { - models.PackageInfo - strarray - }{p, cveIDs} - } - }(pack) - } + // { CVE ID: []pkgAuditResult } + cveIDAdtMap := make(map[string][]pkgAuditResult) + for _, p := range packAdtRslt { + for _, cid := range p.vulnIDCveIDs.cveIDs { + cveIDAdtMap[cid] = append(cveIDAdtMap[cid], p) } } - errs := []error{} - for i := 0; i < len(unsecurePacks); i++ { - o.log.Info(unsecurePacks[i]) - select { - case pair := <-resChan: - pack := pair.PackageInfo - cveIDs := pair.strarray - for _, cveID := range cveIDs { - cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack) - } - o.log.Infof("(%d/%d) Scanned %s-%s : %s", - i+1, len(unsecurePacks), pair.Name, pair.PackageInfo.Version, cveIDs) - case err := <-errChan: - errs = append(errs, err) - case <-timeout: - return nil, fmt.Errorf("Timeout scanPackageCveIDs") - } - } - - if 0 < len(errs) { - return nil, fmt.Errorf("%v", errs) - } - - var cveIDs []string - for k := range cvePackages { + cveIDs := []string{} + for k := range cveIDAdtMap { cveIDs = append(cveIDs, k) } - o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs) - - o.log.Info("Fetching CVE details...") cveDetails, err := cveapi.CveClient.FetchCveDetails(cveIDs) if err != nil { return nil, err } o.log.Info("Done") - for _, detail := range cveDetails { + for _, d := range cveDetails { + packs := []models.PackageInfo{} + for _, r := range cveIDAdtMap[d.CveID] { + packs = append(packs, r.pack) + } + + disAdvs := []models.DistroAdvisory{} + for _, r := range cveIDAdtMap[d.CveID] { + disAdvs = append(disAdvs, models.DistroAdvisory{ + AdvisoryID: r.vulnIDCveIDs.vulnID, + }) + } + cvePacksList = append(cvePacksList, CvePacksInfo{ - CveID: detail.CveID, - CveDetail: detail, - Packs: cvePackages[detail.CveID], - // CvssScore: cinfo.CvssScore(conf.Lang), + CveID: d.CveID, + CveDetail: d, + Packs: packs, + DistroAdvisories: disAdvs, }) } return } -func (o *bsd) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) { - cmd := fmt.Sprintf("pkg audit -F %s | grep CVE-\\w", pack.Name) - cmd = util.PrependProxyEnv(cmd) +func (o *bsd) parsePkgVersion(stdout string) (packs []models.PackageInfo) { + lines := strings.Split(stdout, "\n") + for _, l := range lines { + fields := strings.Fields(l) + if len(fields) < 2 { + continue + } - r := o.ssh(cmd, noSudo) - if !r.isSuccess() { - o.log.Warnf("Failed to %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr) - return nil, nil + packVer := fields[0] + splitted := strings.Split(packVer, "-") + ver := splitted[len(splitted)-1] + name := strings.Join(splitted[:len(splitted)-1], "-") + + switch fields[1] { + case "?", "=": + packs = append(packs, models.PackageInfo{ + Name: name, + Version: ver, + }) + case "<": + candidate := strings.TrimSuffix(fields[6], ")") + packs = append(packs, models.PackageInfo{ + Name: name, + Version: ver, + NewVersion: candidate, + }) + } } - match := regexp.MustCompile("(CVE-\\d{4}-\\d{4})") - return match.FindAllString(r.Stdout, -1), nil + return +} + +type vulnIDCveIDs struct { + vulnID string + cveIDs []string +} + +type pkgAuditResult struct { + pack models.PackageInfo + vulnIDCveIDs vulnIDCveIDs +} + +func (o *bsd) splitIntoBlocks(stdout string) (blocks []string) { + lines := strings.Split(stdout, "\n") + block := []string{} + for _, l := range lines { + if len(strings.TrimSpace(l)) == 0 { + if 0 < len(block) { + blocks = append(blocks, strings.Join(block, "\n")) + block = []string{} + } + continue + } + block = append(block, strings.TrimSpace(l)) + } + if 0 < len(block) { + blocks = append(blocks, strings.Join(block, "\n")) + } + return +} + +func (o *bsd) parseBlock(block string) (packName string, cveIDs []string, vulnID string) { + lines := strings.Split(block, "\n") + for _, l := range lines { + if strings.HasSuffix(l, " is vulnerable:") { + packVer := strings.Fields(l)[0] + splitted := strings.Split(packVer, "-") + packName = strings.Join(splitted[:len(splitted)-1], "-") + } else if strings.HasPrefix(l, "CVE:") { + cveIDs = append(cveIDs, strings.Fields(l)[1]) + } else if strings.HasPrefix(l, "WWW:") { + splitted := strings.Split(l, "/") + vulnID = strings.TrimSuffix(splitted[len(splitted)-1], ".html") + } + } + return } diff --git a/scan/freebsd_test.go b/scan/freebsd_test.go new file mode 100644 index 00000000..2351f2b2 --- /dev/null +++ b/scan/freebsd_test.go @@ -0,0 +1,155 @@ +package scan + +import ( + "reflect" + "testing" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/k0kubun/pp" +) + +func TestParsePkgVersion(t *testing.T) { + var tests = []struct { + in string + expected []models.PackageInfo + }{ + { + `Updating FreeBSD repository catalogue... +FreeBSD repository is up-to-date. +All repositories are up-to-date. +bash-4.2.45 < needs updating (remote has 4.3.42_1) +gettext-0.18.3.1 < needs updating (remote has 0.19.7) +tcl84-8.4.20_2,1 = up-to-date with remote +teTeX-base-3.0_25 ? orphaned: print/teTeX-base`, + + []models.PackageInfo{ + { + Name: "bash", + Version: "4.2.45", + NewVersion: "4.3.42_1", + }, + { + Name: "gettext", + Version: "0.18.3.1", + NewVersion: "0.19.7", + }, + { + Name: "tcl84", + Version: "8.4.20_2,1", + }, + { + Name: "teTeX-base", + Version: "3.0_25", + }, + }, + }, + } + + d := newBsd(config.ServerInfo{}) + for _, tt := range tests { + actual := d.parsePkgVersion(tt.in) + if !reflect.DeepEqual(tt.expected, actual) { + e := pp.Sprintf("%v", tt.expected) + a := pp.Sprintf("%v", actual) + t.Errorf("expected %s, actual %s", e, a) + } + } +} + +func TestSplitIntoBlocks(t *testing.T) { + var tests = []struct { + in string + expected []string + }{ + { + ` +block1 + +block2 +block2 +block2 + +block3 +block3`, + []string{ + `block1`, + "block2\nblock2\nblock2", + "block3\nblock3", + }, + }, + } + + d := newBsd(config.ServerInfo{}) + for _, tt := range tests { + actual := d.splitIntoBlocks(tt.in) + if !reflect.DeepEqual(tt.expected, actual) { + e := pp.Sprintf("%v", tt.expected) + a := pp.Sprintf("%v", actual) + t.Errorf("expected %s, actual %s", e, a) + } + } + +} + +func TestParseBlock(t *testing.T) { + var tests = []struct { + in string + name string + cveIDs []string + vulnID string + }{ + { + + in: `vulnxml file up-to-date +bind96-9.6.3.2.ESV.R10_2 is vulnerable: +bind -- denial of service vulnerability +CVE: CVE-2014-0591 +WWW: https://vuxml.FreeBSD.org/freebsd/cb252f01-7c43-11e3-b0a6-005056a37f68.html`, + name: "bind96", + cveIDs: []string{"CVE-2014-0591"}, + vulnID: "cb252f01-7c43-11e3-b0a6-005056a37f68", + }, + { + in: `bind96-9.6.3.2.ESV.R10_2 is vulnerable: +bind -- denial of service vulnerability +CVE: CVE-2014-8680 +CVE: CVE-2014-8500 +WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html`, + name: "bind96", + cveIDs: []string{"CVE-2014-8680", "CVE-2014-8500"}, + vulnID: "ab3e98d9-8175-11e4-907d-d050992ecde8", + }, + { + in: `hoge-hoge-9.6.3.2.ESV.R10_2 is vulnerable: +bind -- denial of service vulnerability +CVE: CVE-2014-8680 +CVE: CVE-2014-8500 +WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html`, + name: "hoge-hoge", + cveIDs: []string{"CVE-2014-8680", "CVE-2014-8500"}, + vulnID: "ab3e98d9-8175-11e4-907d-d050992ecde8", + }, + { + in: `1 problem(s) in the installed packages found.`, + cveIDs: []string{}, + vulnID: "", + }, + } + + d := newBsd(config.ServerInfo{}) + for _, tt := range tests { + aName, aCveIDs, aVunlnID := d.parseBlock(tt.in) + if tt.name != aName { + t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID) + } + for i := range tt.cveIDs { + if tt.cveIDs[i] != aCveIDs[i] { + t.Errorf("expected cveID: %s, actual %s", tt.cveIDs[i], aCveIDs[i]) + } + } + if tt.vulnID != aVunlnID { + t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID) + } + } +} diff --git a/scan/serverapi.go b/scan/serverapi.go index d3a685d6..70830e81 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -64,10 +64,9 @@ type CvePacksList []CvePacksInfo type CvePacksInfo struct { CveID string CveDetail cve.CveDetail - Packs []models.PackageInfo - DistroAdvisories []models.DistroAdvisory // for Aamazon, RHEL + Packs models.PackageInfoList + DistroAdvisories []models.DistroAdvisory // for Aamazon, RHEL, FreeBSD CpeNames []string - // CvssScore float64 } // FindByCveID find by CVEID diff --git a/scan/sshutil.go b/scan/sshutil.go index d454bb1d..9a927580 100644 --- a/scan/sshutil.go +++ b/scan/sshutil.go @@ -134,11 +134,12 @@ func sshExec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (re logger.Panicf("sudoOpt is invalid. SudoOpt: %v", c.SudoOpt) } } - // set pipefail option. - // http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another - cmd = fmt.Sprintf("set -o pipefail; %s", cmd) - logger.Debugf("Command: %s", - strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1)) + + if c.Family != "FreeBSD" { + // set pipefail option. Bash only + // http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another + cmd = fmt.Sprintf("set -o pipefail; %s", cmd) + } if c.IsContainer() { switch c.Container.Type { @@ -147,6 +148,9 @@ func sshExec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (re } } + logger.Debugf("Command: %s", + strings.Replace(maskPassword(cmd, c.Password), "\n", "", -1)) + var client *ssh.Client client, err = sshConnect(c) defer client.Close()