vuls/scan/debian.go

/* Vuls - Vulnerability Scanner
Copyright (C) 2016  Future Architect, Inc. Japan.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package scan

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/future-architect/vuls/cache"
	"github.com/future-architect/vuls/config"
	"github.com/future-architect/vuls/models"
	"github.com/future-architect/vuls/util"
)

// inherit OsTypeInterface
type debian struct {
	base
}

// NewDebian is constructor
func newDebian(c config.ServerInfo) *debian {
	d := &debian{
		base: base{
			osPackages: osPackages{
				Packages:  models.Packages{},
				VulnInfos: models.VulnInfos{},
			},
		},
	}
	d.log = util.NewCustomLogger(c)
	d.setServerInfo(c)
	return d
}

// Ubuntu, Debian, Raspbian
// https://github.com/serverspec/specinfra/blob/master/lib/specinfra/helper/detect_os/debian.rb
func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err error) {
	deb = newDebian(c)

	if r := exec(c, "ls /etc/debian_version", noSudo); !r.isSuccess() {
		if r.Error != nil {
			return false, deb, nil
		}
		if r.ExitStatus == 255 {
			return false, deb, fmt.Errorf(
				"Unable to connect via SSH. Check SSH settings. %s", r)
		}
		util.Log.Debugf("Not Debian like Linux. %s", r)
		return false, deb, nil
	}

	// Raspbian
	// lsb_release in Raspbian Jessie returns 'Distributor ID: Raspbian'.
	// However, lsb_release in Raspbian Wheezy returns 'Distributor ID: Debian'.
	if r := exec(c, "cat /etc/issue", noSudo); r.isSuccess() {
		//  e.g.
		//  Raspbian GNU/Linux 7 \n \l
		result := strings.Fields(r.Stdout)
		if len(result) > 2 && result[0] == "Raspbian" {
			distro := strings.ToLower(trim(result[0]))
			deb.setDistro(distro, trim(result[2]))
			return true, deb, nil
		}
	}

	if r := exec(c, "lsb_release -ir", noSudo); r.isSuccess() {
		//  e.g.
		//  root@fa3ec524be43:/# lsb_release -ir
		//  Distributor ID:	Ubuntu
		//  Release:	14.04
		re := regexp.MustCompile(`(?s)^Distributor ID:\s*(.+?)\n*Release:\s*(.+?)$`)
		result := re.FindStringSubmatch(trim(r.Stdout))

		if len(result) == 0 {
			deb.setDistro("debian/ubuntu", "unknown")
			util.Log.Warnf(
				"Unknown Debian/Ubuntu version. lsb_release -ir: %s", r)
		} else {
			distro := strings.ToLower(trim(result[1]))
			deb.setDistro(distro, trim(result[2]))
		}
		return true, deb, nil
	}

	if r := exec(c, "cat /etc/lsb-release", noSudo); r.isSuccess() {
		//  e.g.
		//  DISTRIB_ID=Ubuntu
		//  DISTRIB_RELEASE=14.04
		//  DISTRIB_CODENAME=trusty
		//  DISTRIB_DESCRIPTION="Ubuntu 14.04.2 LTS"
		re := regexp.MustCompile(`(?s)^DISTRIB_ID=(.+?)\n*DISTRIB_RELEASE=(.+?)\n.*$`)
		result := re.FindStringSubmatch(trim(r.Stdout))
		if len(result) == 0 {
			util.Log.Warnf(
				"Unknown Debian/Ubuntu. cat /etc/lsb-release: %s", r)
			deb.setDistro("debian/ubuntu", "unknown")
		} else {
			distro := strings.ToLower(trim(result[1]))
			deb.setDistro(distro, trim(result[2]))
		}
		return true, deb, nil
	}

	// Debian
	cmd := "cat /etc/debian_version"
	if r := exec(c, cmd, noSudo); r.isSuccess() {
		deb.setDistro("debian", trim(r.Stdout))
		return true, deb, nil
	}

	util.Log.Debugf("Not Debian like Linux: %s", c.ServerName)
	return false, deb, nil
}

func trim(str string) string {
	return strings.TrimSpace(str)
}

func (o *debian) checkIfSudoNoPasswd() error {
	cmd := util.PrependProxyEnv("apt-get update")
	o.log.Infof("Checking... sudo %s", cmd)
	r := o.exec(cmd, sudo)
	if !r.isSuccess() {
		o.log.Errorf("sudo error on %s", r)
		return fmt.Errorf("Failed to sudo: %s", r)
	}
	o.log.Infof("Sudo... Pass")
	return nil
}

func (o *debian) checkDependencies() error {
	switch o.Distro.Family {
	case "ubuntu", "raspbian":
		return nil

	case "debian":
		// Debian needs aptitude to get changelogs.
		// Because unable to get changelogs via apt-get changelog on Debian.
		if r := o.exec("test -f /usr/bin/aptitude", noSudo); !r.isSuccess() {
			msg := fmt.Sprintf("aptitude is not installed: %s", r)
			o.log.Errorf(msg)
			return fmt.Errorf(msg)
		}
		o.log.Infof("Dependencies... Pass")
		return nil

	default:
		return fmt.Errorf("Not implemented yet: %s", o.Distro)
	}
}

func (o *debian) scanPackages() error {
	installed, upgradable, err := o.scanInstalledPackages()
	if err != nil {
		o.log.Errorf("Failed to scan installed packages")
		return err
	}
	o.setPackages(installed)

	if config.Conf.PackageListOnly {
		return nil
	}

	unsecure, err := o.scanUnsecurePackages(upgradable)
	if err != nil {
		o.log.Errorf("Failed to scan vulnerable packages")
		return err
	}
	o.setVulnInfos(unsecure)
	return nil
}

func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, error) {
	installed := models.Packages{}
	upgradable := models.Packages{}

	r := o.exec("dpkg-query -W", noSudo)
	if !r.isSuccess() {
		return nil, nil, fmt.Errorf("Failed to SSH: %s", r)
	}

	//  e.g.
	//  curl	7.19.7-40.el6_6.4
	//  openldap	2.4.39-8.el6
	lines := strings.Split(r.Stdout, "\n")
	for _, line := range lines {
		if trimmed := strings.TrimSpace(line); len(trimmed) != 0 {
			name, version, err := o.parseScannedPackagesLine(trimmed)
			if err != nil {
				return nil, nil, fmt.Errorf(
					"Debian: Failed to parse package line: %s", line)
			}
			installed[name] = models.Package{
				Name:    name,
				Version: version,
			}
		}
	}

	upgradableNames, err := o.GetUpgradablePackNames()
	if err != nil {
		return nil, nil, err
	}
	for _, name := range upgradableNames {
		for _, pack := range installed {
			if pack.Name == name {
				upgradable[name] = pack
				break
			}
		}
	}

	// Fill the candidate versions of upgradable packages
	err = o.fillCandidateVersion(upgradable)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err)
	}
	installed.MergeNewVersion(upgradable)

	return installed, upgradable, nil
}

var packageLinePattern = regexp.MustCompile(`^([^\t']+)\t(.+)$`)

func (o *debian) parseScannedPackagesLine(line string) (name, version string, err error) {
	result := packageLinePattern.FindStringSubmatch(line)
	if len(result) == 3 {
		// remove :amd64, i386...
		name = result[1]
		if i := strings.IndexRune(name, ':'); i >= 0 {
			name = name[:i]
		}
		version = result[2]
		return
	}

	return "", "", fmt.Errorf("Unknown format: %s", line)
}

func (o *debian) aptGetUpdate() error {
	o.log.Infof("apt-get update...")
	cmd := util.PrependProxyEnv("apt-get update")
	if r := o.exec(cmd, sudo); !r.isSuccess() {
		return fmt.Errorf("Failed to SSH: %s", r)
	}
	return nil
}

func (o *debian) scanUnsecurePackages(upgradable models.Packages) (models.VulnInfos, error) {
	o.aptGetUpdate()

	// Setup changelog cache
	current := cache.Meta{
		Name:   o.getServerInfo().GetServerName(),
		Distro: o.getServerInfo().Distro,
		Packs:  upgradable,
	}

	o.log.Debugf("Ensure changelog cache: %s", current.Name)
	meta, err := o.ensureChangelogCache(current)
	if err != nil {
		return nil, err
	}

	// Collect CVE information of upgradable packages
	vulnInfos, err := o.scanVulnInfos(upgradable, meta)
	if err != nil {
		return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err)
	}

	return vulnInfos, nil
}

func (o *debian) ensureChangelogCache(current cache.Meta) (*cache.Meta, error) {
	// Search from cache
	cached, found, err := cache.DB.GetMeta(current.Name)
	if err != nil {
		return nil, fmt.Errorf(
			"Failed to get meta. Please remove cache.db and then try again. err: %s", err)
	}

	if !found {
		o.log.Debugf("Not found in meta: %s", current.Name)
		err = cache.DB.EnsureBuckets(current)
		if err != nil {
			return nil, fmt.Errorf("Failed to ensure buckets. err: %s", err)
		}
		return &current, nil
	}

	if current.Distro.Family != cached.Distro.Family ||
		current.Distro.Release != cached.Distro.Release {
		o.log.Debugf("Need to refesh meta: %s", current.Name)
		err = cache.DB.EnsureBuckets(current)
		if err != nil {
			return nil, fmt.Errorf("Failed to ensure buckets. err: %s", err)
		}
		return &current, nil

	}

	o.log.Debugf("Reuse meta: %s", current.Name)
	if config.Conf.Debug {
		cache.DB.PrettyPrint(current)
	}
	return &cached, nil
}

func (o *debian) fillCandidateVersion(packages models.Packages) (err error) {
	names := []string{}
	for name := range packages {
		names = append(names, name)
	}
	cmd := fmt.Sprintf("LANGUAGE=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " "))
	r := o.exec(cmd, noSudo)
	if !r.isSuccess() {
		return fmt.Errorf("Failed to SSH: %s", r)
	}
	packChangelog := o.splitAptCachePolicy(r.Stdout)
	for k, v := range packChangelog {
		ver, err := o.parseAptCachePolicy(v, k)
		if err != nil {
			return fmt.Errorf("Failed to parse %s", err)
		}
		pack, ok := packages[k]
		if !ok {
			return fmt.Errorf("Not found: %s", k)
		}
		pack.NewVersion = ver.Candidate
		packages[k] = pack
	}
	return
}

func (o *debian) GetUpgradablePackNames() (packNames []string, err error) {
	cmd := util.PrependProxyEnv("LANGUAGE=en_US.UTF-8 apt-get upgrade --dry-run")
	r := o.exec(cmd, noSudo)
	if r.isSuccess(0, 1) {
		return o.parseAptGetUpgrade(r.Stdout)
	}
	return packNames, fmt.Errorf(
		"Failed to %s. status: %d, stdout: %s, stderr: %s",
		cmd, r.ExitStatus, r.Stdout, r.Stderr)
}

func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, err error) {
	startRe := regexp.MustCompile(`The following packages will be upgraded:`)
	stopRe := regexp.MustCompile(`^(\d+) upgraded.*`)
	startLineFound, stopLineFound := false, false

	lines := strings.Split(stdout, "\n")
	for _, line := range lines {
		if !startLineFound {
			if matche := startRe.MatchString(line); matche {
				startLineFound = true
			}
			continue
		}
		result := stopRe.FindStringSubmatch(line)
		if len(result) == 2 {
			numUpgradablePacks, err := strconv.Atoi(result[1])
			if err != nil {
				return nil, fmt.Errorf(
					"Failed to scan upgradable packages number. line: %s", line)
			}
			if numUpgradablePacks != len(upgradableNames) {
				return nil, fmt.Errorf(
					"Failed to scan upgradable packages, expected: %s, detected: %d",
					result[1], len(upgradableNames))
			}
			stopLineFound = true
			o.log.Debugf("Found the stop line. line: %s", line)
			break
		}
		upgradableNames = append(upgradableNames, strings.Fields(line)...)
	}
	if !startLineFound {
		// no upgrades
		return
	}
	if !stopLineFound {
		// There are upgrades, but not found the stop line.
		return nil, fmt.Errorf("Failed to scan upgradable packages")
	}
	return
}

// DetectedCveID has CveID, Confidence and DetectionMethod fields
// LenientMatching will be true if this vulnerability is not detected by accurate version matching.
// see https://github.com/future-architect/vuls/pull/328
type DetectedCveID struct {
	CveID      string
	Confidence models.Confidence
}

func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) {
	type response struct {
		packName       string
		DetectedCveIDs []DetectedCveID
	}
	resChan := make(chan response, len(upgradablePacks))
	errChan := make(chan error, len(upgradablePacks))
	reqChan := make(chan models.Package, len(upgradablePacks))
	defer close(resChan)
	defer close(errChan)
	defer close(reqChan)

	go func() {
		for _, pack := range upgradablePacks {
			reqChan <- pack
		}
	}()

	timeout := time.After(30 * 60 * time.Second)
	concurrency := 10
	tasks := util.GenWorkers(concurrency)
	for range upgradablePacks {
		tasks <- func() {
			select {
			case pack := <-reqChan:
				func(p models.Package) {
					changelog := o.getChangelogCache(meta, p)
					if 0 < len(changelog) {
						cveIDs, _ := o.getCveIDsFromChangelog(changelog, p.Name, p.Version)
						resChan <- response{p.Name, cveIDs}
						return
					}

					// if the changelog is not in cache or failed to get from local cache,
					// get the changelog of the package via internet.
					// After that, store it in the cache.
					if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
						errChan <- err
					} else {
						resChan <- response{p.Name, cveIDs}
					}
				}(pack)
			}
		}
	}

	// { DetectedCveID{} : [package] }
	cvePackages := make(map[DetectedCveID][]string)
	errs := []error{}
	for i := 0; i < len(upgradablePacks); i++ {
		select {
		case response := <-resChan:
			cves := response.DetectedCveIDs
			for _, cve := range cves {
				packNames, ok := cvePackages[cve]
				if ok {
					packNames = append(packNames, response.packName)
				} else {
					packNames = []string{response.packName}
				}
				cvePackages[cve] = packNames
			}
			o.log.Infof("(%d/%d) Scanned %s: %s",
				i+1, len(upgradablePacks), response.packName, cves)
		case err := <-errChan:
			errs = append(errs, err)
		case <-timeout:
			errs = append(errs, fmt.Errorf("Timeout scanPackageCveIDs"))
		}
	}
	if 0 < len(errs) {
		return nil, fmt.Errorf("%v", errs)
	}

	var cveIDs []DetectedCveID
	for k := range cvePackages {
		cveIDs = append(cveIDs, k)
	}
	o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs)
	vinfos := models.VulnInfos{}
	for cveID, names := range cvePackages {
		vinfos[cveID.CveID] = models.VulnInfo{
			CveID:        cveID.CveID,
			Confidence:   cveID.Confidence,
			PackageNames: names,
		}
	}

	// Update meta package information of changelog cache to the latest one.
	meta.Packs = upgradablePacks
	if err := cache.DB.RefreshMeta(*meta); err != nil {
		return nil, err
	}

	return vinfos, nil
}

func (o *debian) getChangelogCache(meta *cache.Meta, pack models.Package) string {
	cachedPack, found := meta.Packs[pack.Name]
	if !found {
		o.log.Debugf("Not found: %s", pack.Name)
		return ""
	}

	if cachedPack.NewVersion != pack.NewVersion {
		o.log.Debugf("Expired: %s, cache: %s, new: %s",
			pack.Name, cachedPack.NewVersion, pack.NewVersion)
		return ""
	}
	changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name)
	if err != nil {
		o.log.Warnf("Failed to get changelog. bucket: %s, key:%s, err: %s",
			meta.Name, pack.Name, err)
		return ""
	}
	if len(changelog) == 0 {
		o.log.Debugf("Empty string: %s", pack.Name)
		return ""
	}

	o.log.Debugf("Hit: %s, %s, cache: %s, new: %s len: %d, %s...",
		meta.Name, pack.Name, cachedPack.NewVersion, pack.NewVersion, len(changelog), util.Truncate(changelog, 30))
	return changelog
}

func (o *debian) scanPackageCveIDs(pack models.Package) ([]DetectedCveID, error) {
	cmd := ""
	switch o.Distro.Family {
	case "ubuntu", "raspbian":
		cmd = fmt.Sprintf(`PAGER=cat apt-get -q=2 changelog %s`, pack.Name)
	case "debian":
		cmd = fmt.Sprintf(`PAGER=cat aptitude -q=2 changelog %s`, pack.Name)
	}
	cmd = util.PrependProxyEnv(cmd)

	r := o.exec(cmd, noSudo)
	if !r.isSuccess() {
		o.log.Warnf("Failed to SSH: %s", r)
		// Ignore this Error.
		return nil, nil
	}

	stdout := strings.Replace(r.Stdout, "\r", "", -1)
	cveIDs, clog := o.getCveIDsFromChangelog(
		stdout, pack.Name, pack.Version)

	if clog.Method != models.FailedToGetChangelog {
		err := cache.DB.PutChangelog(o.getServerInfo().GetServerName(), pack.Name, clog.Contents)
		if err != nil {
			return nil, fmt.Errorf("Failed to put changelog into cache")
		}
	}

	// No error will be returned. Only logging.
	return cveIDs, nil
}

// Debian Version Numbers
// https://readme.phys.ethz.ch/documentation/debian_version_numbers/
func (o *debian) getCveIDsFromChangelog(
	changelog, name, ver string) ([]DetectedCveID, models.Changelog) {

	if cveIDs, relevant, err := o.parseChangelog(
		changelog, name, ver, models.ChangelogExactMatch); err == nil {
		return cveIDs, relevant
	}

	var verAfterColon string
	var err error

	splittedByColon := strings.Split(ver, ":")
	if 1 < len(splittedByColon) {
		verAfterColon = splittedByColon[1]
		if cveIDs, relevant, err := o.parseChangelog(
			changelog, name, verAfterColon, models.ChangelogLenientMatch); err == nil {
			return cveIDs, relevant
		}
	}

	delim := []string{"+", "~", "build"}
	switch o.Distro.Family {
	case "ubuntu":
		delim = append(delim, "ubuntu")
	case "debian":
	case "Raspbian":
	}

	for _, d := range delim {
		ss := strings.Split(ver, d)
		if 1 < len(ss) {
			if cveIDs, relevant, err := o.parseChangelog(
				changelog, name, ss[0], models.ChangelogLenientMatch); err == nil {
				return cveIDs, relevant
			}
		}

		ss = strings.Split(verAfterColon, d)
		if 1 < len(ss) {
			if cveIDs, relevant, err := o.parseChangelog(
				changelog, name, ss[0], models.ChangelogLenientMatch); err == nil {
				return cveIDs, relevant
			}
		}
	}

	// Only logging the error.
	o.log.Error(err)

	pack := o.Packages[name]
	pack.Changelog = models.Changelog{
		Contents: "",
		Method:   models.FailedToFindVersionInChangelog,
	}
	//TODO Mutex
	o.Packages[name] = pack

	// If the version is not in changelog, return entire changelog to put into cache
	return []DetectedCveID{}, models.Changelog{
		Contents: changelog,
		Method:   models.FailedToFindVersionInChangelog,
	}
}

var cveRe = regexp.MustCompile(`(CVE-\d{4}-\d{4,})`)

// Collect CVE-IDs included in the changelog.
// The version which specified in argument(versionOrLater) is excluded.
func (o *debian) parseChangelog(changelog, name, ver string, confidence models.Confidence) ([]DetectedCveID, models.Changelog, error) {
	buf, cveIDs := []string{}, []string{}
	stopRe := regexp.MustCompile(fmt.Sprintf(`\(%s\)`, regexp.QuoteMeta(ver)))
	stopLineFound := false
	lines := strings.Split(changelog, "\n")
	for _, line := range lines {
		buf = append(buf, line)
		if match := stopRe.MatchString(line); match {
			//  o.log.Debugf("Found the stop line: %s", line)
			stopLineFound = true
			break
		} else if matches := cveRe.FindAllString(line, -1); 0 < len(matches) {
			for _, m := range matches {
				cveIDs = util.AppendIfMissing(cveIDs, m)
			}
		}
	}
	if !stopLineFound {
		return nil, models.Changelog{
				Contents: "",
				Method:   models.FailedToFindVersionInChangelog,
			}, fmt.Errorf(
				"Failed to scan CVE IDs. The version is not in changelog. name: %s, version: %s",
				name,
				ver,
			)
	}

	clog := models.Changelog{
		Contents: strings.Join(buf, "\n"),
		Method:   string(confidence.DetectionMethod),
	}

	pack := o.Packages[name]
	pack.Changelog = clog
	o.Packages[name] = pack

	cves := []DetectedCveID{}
	for _, id := range cveIDs {
		cves = append(cves, DetectedCveID{id, confidence})
	}

	return cves, clog, nil
}

func (o *debian) splitAptCachePolicy(stdout string) map[string]string {
	re := regexp.MustCompile(`(?m:^[^ \t]+:\r?\n)`)
	ii := re.FindAllStringIndex(stdout, -1)
	ri := []int{}
	for i := len(ii) - 1; 0 <= i; i-- {
		ri = append(ri, ii[i][0])
	}
	splitted := []string{}
	lasti := len(stdout)
	for _, i := range ri {
		splitted = append(splitted, stdout[i:lasti])
		lasti = i
	}

	packChangelog := map[string]string{}
	for _, r := range splitted {
		packName := r[:strings.Index(r, ":")]
		packChangelog[packName] = r
	}
	return packChangelog
}

type packCandidateVer struct {
	Name      string
	Installed string
	Candidate string
}

// parseAptCachePolicy the stdout of parse pat-get cache policy
func (o *debian) parseAptCachePolicy(stdout, name string) (packCandidateVer, error) {
	ver := packCandidateVer{Name: name}
	lines := strings.Split(stdout, "\n")
	for _, line := range lines {
		fields := strings.Fields(line)
		if len(fields) != 2 {
			continue
		}
		switch fields[0] {
		case "Installed:":
			ver.Installed = fields[1]
		case "Candidate:":
			ver.Candidate = fields[1]
			return ver, nil
		default:
			// nop
		}
	}
	return ver, fmt.Errorf("Unknown Format: %s", stdout)
}