High speed scan on Ubuntu/Debian

2016-09-08 19:18:49 +09:00
parent 2afe2d2640
commit dd1d3a05fa
25 changed files with 1035 additions and 317 deletions
--- a/scan/base.go
+++ b/scan/base.go
@@ -32,9 +32,8 @@ import (

 type base struct {
 	ServerInfo config.ServerInfo
+	Distro     config.Distro

-	Family   string
-	Release  string
 	Platform models.Platform
 	osPackages

@@ -54,13 +53,20 @@ func (l base) getServerInfo() config.ServerInfo {
 	return l.ServerInfo
 }

-func (l *base) setDistributionInfo(fam, rel string) {
-	l.Family = fam
-	l.Release = rel
+func (l *base) setDistro(fam, rel string) {
+	d := config.Distro{
+		Family:  fam,
+		Release: rel,
+	}
+	l.Distro = d
+
+	s := l.getServerInfo()
+	s.Distro = d
+	l.setServerInfo(s)
 }

-func (l base) getDistributionInfo() string {
-	return fmt.Sprintf("%s %s", l.Family, l.Release)
+func (l base) getDistro() config.Distro {
+	return l.Distro
 }

 func (l *base) setPlatform(p models.Platform) {
@@ -250,8 +256,8 @@ func (l *base) convertToModel() (models.ScanResult, error) {
 	return models.ScanResult{
 		ServerName:  l.ServerInfo.ServerName,
 		ScannedAt:   time.Now(),
-		Family:      l.Family,
-		Release:     l.Release,
+		Family:      l.Distro.Family,
+		Release:     l.Distro.Release,
 		Container:   container,
 		Platform:    l.Platform,
 		KnownCves:   scoredCves,
--- a/scan/base_test.go
+++ b/scan/base_test.go
@@ -70,8 +70,9 @@ func TestIsAwsInstanceID(t *testing.T) {
 		{"no data", false},
 	}

+	r := newRedhat(config.ServerInfo{})
 	for _, tt := range tests {
-		actual := isAwsInstanceID(tt.in)
+		actual := r.isAwsInstanceID(tt.in)
 		if tt.expected != actual {
 			t.Errorf("expected %t, actual %t, str: %s", tt.expected, actual, tt.in)
 		}
--- a/scan/debian.go
+++ b/scan/debian.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 	"time"

+	"github.com/future-architect/vuls/cache"
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/cveapi"
 	"github.com/future-architect/vuls/models"
@@ -39,6 +40,7 @@ type debian struct {
 func newDebian(c config.ServerInfo) *debian {
 	d := &debian{}
 	d.log = util.NewCustomLogger(c)
+	d.setServerInfo(c)
 	return d
 }

@@ -46,7 +48,6 @@ func newDebian(c config.ServerInfo) *debian {
 // https://github.com/serverspec/specinfra/blob/master/lib/specinfra/helper/detect_os/debian.rb
 func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err error) {
 	deb = newDebian(c)
-	deb.setServerInfo(c)

 	if r := sshExec(c, "ls /etc/debian_version", noSudo); !r.isSuccess() {
 		if r.Error != nil {
@@ -69,12 +70,12 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
 		result := re.FindStringSubmatch(trim(r.Stdout))

 		if len(result) == 0 {
-			deb.setDistributionInfo("debian/ubuntu", "unknown")
+			deb.setDistro("debian/ubuntu", "unknown")
 			Log.Warnf(
 				"Unknown Debian/Ubuntu version. lsb_release -ir: %s", r)
 		} else {
 			distro := strings.ToLower(trim(result[1]))
-			deb.setDistributionInfo(distro, trim(result[2]))
+			deb.setDistro(distro, trim(result[2]))
 		}
 		return true, deb, nil
 	}
@@ -90,10 +91,10 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
 		if len(result) == 0 {
 			Log.Warnf(
 				"Unknown Debian/Ubuntu. cat /etc/lsb-release: %s", r)
-			deb.setDistributionInfo("debian/ubuntu", "unknown")
+			deb.setDistro("debian/ubuntu", "unknown")
 		} else {
 			distro := strings.ToLower(trim(result[1]))
-			deb.setDistributionInfo(distro, trim(result[2]))
+			deb.setDistro(distro, trim(result[2]))
 		}
 		return true, deb, nil
 	}
@@ -101,7 +102,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
 	// Debian
 	cmd := "cat /etc/debian_version"
 	if r := sshExec(c, cmd, noSudo); r.isSuccess() {
-		deb.setDistributionInfo("debian", trim(r.Stdout))
+		deb.setDistro("debian", trim(r.Stdout))
 		return true, deb, nil
 	}

@@ -133,7 +134,7 @@ func (o *debian) install() error {
 		return fmt.Errorf(msg)
 	}

-	if o.Family == "debian" {
+	if o.Distro.Family == "debian" {
 		// install aptitude
 		cmd = util.PrependProxyEnv("apt-get install --force-yes -y aptitude")
 		if r := o.ssh(cmd, sudo); !r.isSuccess() {
@@ -208,7 +209,7 @@ func (o *debian) parseScannedPackagesLine(line string) (name, version string, er
 }

 func (o *debian) checkRequiredPackagesInstalled() error {
-	if o.Family == "debian" {
+	if o.Distro.Family == "debian" {
 		if r := o.ssh("test -f /usr/bin/aptitude", noSudo); !r.isSuccess() {
 			msg := fmt.Sprintf("aptitude is not installed: %s", r)
 			o.log.Errorf(msg)
@@ -218,9 +219,8 @@ func (o *debian) checkRequiredPackagesInstalled() error {
 	return nil
 }

-//TODO return whether already expired.
 func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInfo, error) {
-	//  cmd := prependProxyEnv(conf.HTTPProxy, "apt-get update | cat; echo 1")
+	o.log.Infof("apt-get update...")
 	cmd := util.PrependProxyEnv("apt-get update")
 	if r := o.ssh(cmd, sudo); !r.isSuccess() {
 		return nil, fmt.Errorf("Failed to SSH: %s", r)
@@ -241,12 +241,21 @@ func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInf
 			}
 		}
 	}
-
 	unsecurePacks, err = o.fillCandidateVersion(unsecurePacks)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err)
 	}

+	current := cache.Meta{
+		Name:   o.getServerInfo().ServerName,
+		Distro: o.getServerInfo().Distro,
+		Packs:  unsecurePacks,
+	}
+	o.log.Debugf("Ensure changelog cache: %s", current.Name)
+	if err := o.ensureChangelogCache(current); err != nil {
+		return nil, err
+	}
+
 	// Collect CVE information of upgradable packages
 	cvePacksInfos, err := o.scanPackageCveInfos(unsecurePacks)
 	if err != nil {
@@ -256,63 +265,61 @@ func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInf
 	return cvePacksInfos, nil
 }

-func (o *debian) fillCandidateVersion(packs []models.PackageInfo) ([]models.PackageInfo, error) {
-	reqChan := make(chan models.PackageInfo, len(packs))
-	resChan := make(chan models.PackageInfo, len(packs))
-	errChan := make(chan error, len(packs))
-	defer close(resChan)
-	defer close(errChan)
-	defer close(reqChan)
-
-	go func() {
-		for _, pack := range packs {
-			reqChan <- pack
+func (o *debian) ensureChangelogCache(current cache.Meta) error {
+	// Search from cache
+	old, found, err := cache.DB.GetMeta(current.Name)
+	if err != nil {
+		return fmt.Errorf("Failed to get meta. err: %s", err)
+	}
+	if !found {
+		o.log.Debugf("Not found in meta: %s", current.Name)
+		err = cache.DB.EnsureBuckets(current)
+		if err != nil {
+			return fmt.Errorf("Failed to ensure buckets. err: %s", err)
 		}
-	}()
-
-	timeout := time.After(5 * 60 * time.Second)
-	concurrency := 5
-	tasks := util.GenWorkers(concurrency)
-	for range packs {
-		tasks <- func() {
-			select {
-			case pack := <-reqChan:
-				func(p models.PackageInfo) {
-					cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", p.Name)
-					r := o.ssh(cmd, sudo)
-					if !r.isSuccess() {
-						errChan <- fmt.Errorf("Failed to SSH: %s.", r)
-						return
-					}
-					ver, err := o.parseAptCachePolicy(r.Stdout, p.Name)
-					if err != nil {
-						errChan <- fmt.Errorf("Failed to parse %s", err)
-					}
-					p.NewVersion = ver.Candidate
-					resChan <- p
-				}(pack)
+	} else {
+		if current.Distro.Family != old.Distro.Family ||
+			current.Distro.Release != old.Distro.Release {
+			o.log.Debugf("Need to refesh meta: %s", current.Name)
+			err = cache.DB.EnsureBuckets(current)
+			if err != nil {
+				return fmt.Errorf("Failed to ensure buckets. err: %s", err)
 			}
+		} else {
+			o.log.Debugf("Reuse meta: %s", current.Name)
 		}
 	}

-	errs := []error{}
-	result := []models.PackageInfo{}
-	for i := 0; i < len(packs); i++ {
-		select {
-		case pack := <-resChan:
-			result = append(result, pack)
-			o.log.Infof("(%d/%d) Upgradable: %s-%s -> %s",
-				i+1, len(packs), pack.Name, pack.Version, pack.NewVersion)
-		case err := <-errChan:
-			errs = append(errs, err)
-		case <-timeout:
-			return nil, fmt.Errorf("Timeout fillCandidateVersion")
+	if config.Conf.Debug {
+		cache.DB.PrettyPrint(current)
+	}
+	return nil
+}
+
+func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []models.PackageInfo, err error) {
+	names := []string{}
+	for _, p := range before {
+		names = append(names, p.Name)
+	}
+	cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " "))
+	r := o.ssh(cmd, sudo)
+	if !r.isSuccess() {
+		return nil, fmt.Errorf("Failed to SSH: %s.", r)
+	}
+	packChangelog := o.splitAptCachePolicy(r.Stdout)
+	for k, v := range packChangelog {
+		ver, err := o.parseAptCachePolicy(v, k)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to parse %s", err)
 		}
+		p, found := before.FindByName(k)
+		if !found {
+			return nil, fmt.Errorf("Not found: %s", k)
+		}
+		p.NewVersion = ver.Candidate
+		filled = append(filled, p)
 	}
-	if 0 < len(errs) {
-		return nil, fmt.Errorf("%v", errs)
-	}
-	return result, nil
+	return
 }

 func (o *debian) GetUpgradablePackNames() (packNames []string, err error) {
@@ -369,9 +376,11 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er
 }

 func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePacksList CvePacksList, err error) {
-
-	// { CVE ID: [packageInfo] }
-	cvePackages := make(map[string][]models.PackageInfo)
+	meta := cache.Meta{
+		Name:   o.getServerInfo().ServerName,
+		Distro: o.getServerInfo().Distro,
+		Packs:  unsecurePacks,
+	}

 	type strarray []string
 	resChan := make(chan struct {
@@ -398,6 +407,18 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
 			select {
 			case pack := <-reqChan:
 				func(p models.PackageInfo) {
+					changelog := o.getChangelogCache(meta, p)
+					if 0 < len(changelog) {
+						cveIDs := o.getCveIDFromChangelog(changelog, p.Name, p.Version)
+						resChan <- struct {
+							models.PackageInfo
+							strarray
+						}{p, cveIDs}
+						return
+					}
+
+					// if the changelog is not in cache or failed to get from local cache,
+					// get the changelog of the package via internet.
 					if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
 						errChan <- err
 					} else {
@@ -411,6 +432,8 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
 		}
 	}

+	// { CVE ID: [packageInfo] }
+	cvePackages := make(map[string][]models.PackageInfo)
 	errs := []error{}
 	for i := 0; i < len(unsecurePacks); i++ {
 		select {
@@ -429,7 +452,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
 			return nil, fmt.Errorf("Timeout scanPackageCveIDs")
 		}
 	}
-
 	if 0 < len(errs) {
 		return nil, fmt.Errorf("%v", errs)
 	}
@@ -438,7 +460,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
 	for k := range cvePackages {
 		cveIDs = append(cveIDs, k)
 	}
-
 	o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs)

 	o.log.Info("Fetching CVE details...")
@@ -459,9 +480,32 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
 	return
 }

+func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) string {
+	cachedPack, found := meta.FindPack(pack.Name)
+	if !found {
+		return ""
+	}
+	if cachedPack.NewVersion != pack.NewVersion {
+		return ""
+	}
+	changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name)
+	if err != nil {
+		o.log.Warnf("Failed to get chnagelog. bucket: %s, key:%s, err: %s",
+			meta.Name, pack.Name, err)
+		return ""
+	}
+	if len(changelog) == 0 {
+		return ""
+	}
+
+	o.log.Debugf("Cache hit: %s, len: %d, %s...",
+		meta.Name, len(changelog), util.Truncate(changelog, 30))
+	return changelog
+}
+
 func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) {
 	cmd := ""
-	switch o.Family {
+	switch o.Distro.Family {
 	case "ubuntu":
 		cmd = fmt.Sprintf(`apt-get changelog %s | grep '\(urgency\|CVE\)'`, pack.Name)
 	case "debian":
@@ -476,36 +520,38 @@ func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) {
 		return nil, nil

 	}
+	err := cache.DB.PutChangelog(o.getServerInfo().ServerName, pack.Name, r.Stdout)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to put changelog into cache")
+	}
 	// No error will be returned. Only logging.
-	return o.getCveIDParsingChangelog(r.Stdout, pack.Name, pack.Version)
+	return o.getCveIDFromChangelog(r.Stdout, pack.Name, pack.Version), nil
 }

-func (o *debian) getCveIDParsingChangelog(changelog string,
-	packName string, versionOrLater string) (cveIDs []string, err error) {
+func (o *debian) getCveIDFromChangelog(changelog string,
+	packName string, versionOrLater string) []string {

-	cveIDs, err = o.parseChangelog(changelog, packName, versionOrLater)
-	if err == nil {
-		return
+	if cveIDs, err := o.parseChangelog(changelog, packName, versionOrLater); err == nil {
+		return cveIDs
 	}

 	ver := strings.Split(versionOrLater, "ubuntu")[0]
-	cveIDs, err = o.parseChangelog(changelog, packName, ver)
-	if err == nil {
-		return
+	if cveIDs, err := o.parseChangelog(changelog, packName, ver); err == nil {
+		return cveIDs
 	}

 	splittedByColon := strings.Split(versionOrLater, ":")
 	if 1 < len(splittedByColon) {
 		ver = splittedByColon[1]
 	}
-	cveIDs, err = o.parseChangelog(changelog, packName, ver)
+	cveIDs, err := o.parseChangelog(changelog, packName, ver)
 	if err == nil {
-		return
+		return cveIDs
 	}

 	// Only logging the error.
 	o.log.Error(err)
-	return []string{}, nil
+	return []string{}
 }

 // Collect CVE-IDs included in the changelog.
@@ -538,6 +584,29 @@ func (o *debian) parseChangelog(changelog string,
 	return
 }

+func (o *debian) splitAptCachePolicy(stdout string) map[string]string {
+	//  re := regexp.MustCompile(`(?m:^[^ \t]+:$)`)
+	re := regexp.MustCompile(`(?m:^[^ \t]+:\r\n)`)
+	ii := re.FindAllStringIndex(stdout, -1)
+	ri := []int{}
+	for i := len(ii) - 1; 0 <= i; i-- {
+		ri = append(ri, ii[i][0])
+	}
+	splitted := []string{}
+	lasti := len(stdout)
+	for _, i := range ri {
+		splitted = append(splitted, stdout[i:lasti])
+		lasti = i
+	}
+
+	packChangelog := map[string]string{}
+	for _, r := range splitted {
+		packName := r[:strings.Index(r, ":")]
+		packChangelog[packName] = r
+	}
+	return packChangelog
+}
+
 type packCandidateVer struct {
 	Name      string
 	Installed string
--- a/scan/debian_test.go
+++ b/scan/debian_test.go
@@ -18,10 +18,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package scan

 import (
+	"os"
 	"reflect"
 	"testing"

+	"github.com/Sirupsen/logrus"
+	"github.com/future-architect/vuls/cache"
 	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/models"
 	"github.com/k0kubun/pp"
 )

@@ -183,7 +187,7 @@ util-linux (2.26.2-6) unstable; urgency=medium`,

 	d := newDebian(config.ServerInfo{})
 	for _, tt := range tests {
-		actual, _ := d.getCveIDParsingChangelog(tt.in[2], tt.in[0], tt.in[1])
+		actual := d.getCveIDFromChangelog(tt.in[2], tt.in[0], tt.in[1])
 		if len(actual) != len(tt.expected) {
 			t.Errorf("Len of return array are'nt same. expected %#v, actual %#v", tt.expected, actual)
 			t.Errorf(pp.Sprintf("%s", tt.in))
@@ -195,13 +199,6 @@ util-linux (2.26.2-6) unstable; urgency=medium`,
 			}
 		}
 	}
-
-	for _, tt := range tests {
-		_, err := d.getCveIDParsingChangelog(tt.in[2], tt.in[0], "version number do'nt match case")
-		if err != nil {
-			t.Errorf("Returning error is unexpected")
-		}
-	}
 }

 func TestGetUpdatablePackNames(t *testing.T) {
@@ -520,6 +517,95 @@ Calculating upgrade... Done
 	}
 }

+func TestGetChangelogCache(t *testing.T) {
+	const servername = "server1"
+	pack := models.PackageInfo{
+		Name:       "apt",
+		Version:    "1.0.0",
+		NewVersion: "1.0.1",
+	}
+	var meta = cache.Meta{
+		Name: servername,
+		Distro: config.Distro{
+			Family:  "ubuntu",
+			Release: "16.04",
+		},
+		Packs: []models.PackageInfo{pack},
+	}
+
+	const path = "/tmp/vuls-test-cache-11111111.db"
+	log := logrus.NewEntry(&logrus.Logger{})
+	if err := cache.SetupBolt(path, log); err != nil {
+		t.Errorf("Failed to setup bolt: %s", err)
+	}
+	defer os.Remove(path)
+
+	if err := cache.DB.EnsureBuckets(meta); err != nil {
+		t.Errorf("Failed to ensure buckets: %s", err)
+	}
+
+	d := newDebian(config.ServerInfo{})
+	actual := d.getChangelogCache(meta, pack)
+	if actual != "" {
+		t.Errorf("Failed to get empty stirng from cache:")
+	}
+
+	clog := "changelog-text"
+	if err := cache.DB.PutChangelog(servername, "apt", clog); err != nil {
+		t.Errorf("Failed to put changelog: %s", err)
+	}
+
+	actual = d.getChangelogCache(meta, pack)
+	if actual != clog {
+		t.Errorf("Failed to get changelog from cache: %s", actual)
+	}
+
+	// increment a version of the pack
+	pack.NewVersion = "1.0.2"
+	actual = d.getChangelogCache(meta, pack)
+	if actual != "" {
+		t.Errorf("The changelog is not invalidated: %s", actual)
+	}
+
+	// change a name of the pack
+	pack.Name = "bash"
+	actual = d.getChangelogCache(meta, pack)
+	if actual != "" {
+		t.Errorf("The changelog is not invalidated: %s", actual)
+	}
+}
+
+func TestSplitAptCachePolicy(t *testing.T) {
+	var tests = []struct {
+		stdout   string
+		expected map[string]string
+	}{
+		// This function parse apt-cache policy by using Regexp multi-line mode.
+		// So, test data includes "\r\n"
+		{
+			"apt:\r\n  Installed: 1.2.6\r\n  Candidate: 1.2.12~ubuntu16.04.1\r\n  Version table:\r\n     1.2.12~ubuntu16.04.1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n     1.2.10ubuntu1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n        100 /var/lib/dpkg/status\r\napt-utils:\r\n  Installed: 1.2.6\r\n  Candidate: 1.2.12~ubuntu16.04.1\r\n  Version table:\r\n     1.2.12~ubuntu16.04.1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n     1.2.10ubuntu1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n        100 /var/lib/dpkg/status\r\nbase-files:\r\n  Installed: 9.4ubuntu3\r\n  Candidate: 9.4ubuntu4.2\r\n  Version table:\r\n     9.4ubuntu4.2 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n     9.4ubuntu4 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 9.4ubuntu3 100\r\n        100 /var/lib/dpkg/status\r\n",
+
+			map[string]string{
+				"apt": "apt:\r\n  Installed: 1.2.6\r\n  Candidate: 1.2.12~ubuntu16.04.1\r\n  Version table:\r\n     1.2.12~ubuntu16.04.1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n     1.2.10ubuntu1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n        100 /var/lib/dpkg/status\r\n",
+
+				"apt-utils": "apt-utils:\r\n  Installed: 1.2.6\r\n  Candidate: 1.2.12~ubuntu16.04.1\r\n  Version table:\r\n     1.2.12~ubuntu16.04.1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n     1.2.10ubuntu1 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 1.2.6 100\r\n        100 /var/lib/dpkg/status\r\n",
+
+				"base-files": "base-files:\r\n  Installed: 9.4ubuntu3\r\n  Candidate: 9.4ubuntu4.2\r\n  Version table:\r\n     9.4ubuntu4.2 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages\r\n     9.4ubuntu4 500\r\n        500 http://archive.ubuntu.com/ubuntu xenial/main amd64 Packages\r\n *** 9.4ubuntu3 100\r\n        100 /var/lib/dpkg/status\r\n",
+			},
+		},
+	}
+
+	d := newDebian(config.ServerInfo{})
+	for _, tt := range tests {
+		actual := d.splitAptCachePolicy(tt.stdout)
+		if !reflect.DeepEqual(tt.expected, actual) {
+			e := pp.Sprintf("%v", tt.expected)
+			a := pp.Sprintf("%v", actual)
+			t.Errorf("expected %s, actual %s", e, a)
+		}
+	}
+}
+
 func TestParseAptCachePolicy(t *testing.T) {

 	var tests = []struct {
--- a/scan/freebsd.go
+++ b/scan/freebsd.go
@@ -1,3 +1,20 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package scan

 import (
@@ -19,18 +36,18 @@ type bsd struct {
 func newBsd(c config.ServerInfo) *bsd {
 	d := &bsd{}
 	d.log = util.NewCustomLogger(c)
+	d.setServerInfo(c)
 	return d
 }

 //https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/freebsd.rb
 func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) {
 	bsd = newBsd(c)
-	c.Family = "FreeBSD"
 	if r := sshExec(c, "uname", noSudo); r.isSuccess() {
 		if strings.Contains(r.Stdout, "FreeBSD") == true {
 			if b := sshExec(c, "uname -r", noSudo); b.isSuccess() {
-				bsd.setDistributionInfo("FreeBSD", strings.TrimSpace(b.Stdout))
-				bsd.setServerInfo(c)
+				rel := strings.TrimSpace(b.Stdout)
+				bsd.setDistro("FreeBSD", rel)
 				return true, bsd
 			}
 		}
--- a/scan/redhat.go
+++ b/scan/redhat.go
@@ -42,16 +42,16 @@ type redhat struct {
 func newRedhat(c config.ServerInfo) *redhat {
 	r := &redhat{}
 	r.log = util.NewCustomLogger(c)
+	r.setServerInfo(c)
 	return r
 }

 // https://github.com/serverspec/specinfra/blob/master/lib/specinfra/helper/detect_os/redhat.rb
 func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) {
 	red = newRedhat(c)
-	red.setServerInfo(c)

 	if r := sshExec(c, "ls /etc/fedora-release", noSudo); r.isSuccess() {
-		red.setDistributionInfo("fedora", "unknown")
+		red.setDistro("fedora", "unknown")
 		Log.Warn("Fedora not tested yet: %s", r)
 		return true, red
 	}
@@ -72,9 +72,9 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) {
 			release := result[2]
 			switch strings.ToLower(result[1]) {
 			case "centos", "centos linux":
-				red.setDistributionInfo("centos", release)
+				red.setDistro("centos", release)
 			default:
-				red.setDistributionInfo("rhel", release)
+				red.setDistro("rhel", release)
 			}
 			return true, red
 		}
@@ -90,7 +90,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) {
 				release = fields[4]
 			}
 		}
-		red.setDistributionInfo(family, release)
+		red.setDistro(family, release)
 		return true, red
 	}

@@ -113,7 +113,7 @@ func (o *redhat) checkIfSudoNoPasswd() error {
 // CentOS 7 ... yum-plugin-changelog
 // RHEL, Amazon ... no additinal packages needed
 func (o *redhat) install() error {
-	switch o.Family {
+	switch o.Distro.Family {
 	case "rhel", "amazon":
 		o.log.Infof("Nothing to do")
 		return nil
@@ -123,14 +123,13 @@ func (o *redhat) install() error {
 }

 func (o *redhat) installYumChangelog() error {
-	if o.Family == "centos" {
+	if o.Distro.Family == "centos" {
 		var majorVersion int
-		if 0 < len(o.Release) {
-			majorVersion, _ = strconv.Atoi(strings.Split(o.Release, ".")[0])
+		if 0 < len(o.Distro.Release) {
+			majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0])
 		} else {
 			return fmt.Errorf(
-				"Not implemented yet. family: %s, release: %s",
-				o.Family, o.Release)
+				"Not implemented yet: %s", o.Distro)
 		}

 		var packName = ""
@@ -157,12 +156,12 @@ func (o *redhat) installYumChangelog() error {
 }

 func (o *redhat) checkRequiredPackagesInstalled() error {
-	if o.Family == "centos" {
+	if o.Distro.Family == "centos" {
 		var majorVersion int
-		if 0 < len(o.Release) {
-			majorVersion, _ = strconv.Atoi(strings.Split(o.Release, ".")[0])
+		if 0 < len(o.Distro.Release) {
+			majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0])
 		} else {
-			msg := fmt.Sprintf("Not implemented yet. family: %s, release: %s", o.Family, o.Release)
+			msg := fmt.Sprintf("Not implemented yet: %s", o.Distro)
 			o.log.Errorf(msg)
 			return fmt.Errorf(msg)
 		}
@@ -240,7 +239,7 @@ func (o *redhat) parseScannedPackagesLine(line string) (models.PackageInfo, erro
 }

 func (o *redhat) scanUnsecurePackages() ([]CvePacksInfo, error) {
-	if o.Family != "centos" {
+	if o.Distro.Family != "centos" {
 		// Amazon, RHEL has yum updateinfo as default
 		// yum updateinfo can collenct vendor advisory information.
 		return o.scanUnsecurePackagesUsingYumPluginSecurity()
@@ -460,12 +459,10 @@ func (o *redhat) getChangelogCVELines(rpm2changelog map[string]*string, packInfo

 func (o *redhat) parseAllChangelog(allChangelog string) (map[string]*string, error) {
 	var majorVersion int
-	if 0 < len(o.Release) && o.Family == "centos" {
-		majorVersion, _ = strconv.Atoi(strings.Split(o.Release, ".")[0])
+	if 0 < len(o.Distro.Release) && o.Distro.Family == "centos" {
+		majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0])
 	} else {
-		return nil, fmt.Errorf(
-			"Not implemented yet. family: %s, release: %s",
-			o.Family, o.Release)
+		return nil, fmt.Errorf("Not implemented yet: %s", o.getDistro())
 	}

 	orglines := strings.Split(allChangelog, "\n")
@@ -569,7 +566,7 @@ type distroAdvisoryCveIDs struct {
 // Scaning unsecure packages using yum-plugin-security.
 // Amazon, RHEL
 func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (CvePacksList, error) {
-	if o.Family == "centos" {
+	if o.Distro.Family == "centos" {
 		// CentOS has no security channel.
 		// So use yum check-update && parse changelog
 		return CvePacksList{}, fmt.Errorf(
@@ -717,7 +714,7 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID

 		switch sectionState {
 		case Header:
-			switch o.Family {
+			switch o.Distro.Family {
 			case "centos":
 				// CentOS has no security channel.
 				// So use yum check-update && parse changelog
@@ -964,7 +961,7 @@ func (o *redhat) clone() osTypeInterface {
 }

 func (o *redhat) sudo() bool {
-	switch o.Family {
+	switch o.Distro.Family {
 	case "amazon":
 		return false
 	default:
--- a/scan/redhat_test.go
+++ b/scan/redhat_test.go
@@ -451,7 +451,7 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of
 	updated, _ := time.Parse("2006-01-02", "2015-09-04")

 	r := newRedhat(config.ServerInfo{})
-	r.Family = "redhat"
+	r.Distro = config.Distro{Family: "redhat"}

 	var tests = []struct {
 		in  string
@@ -511,7 +511,7 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of
 func TestParseYumUpdateinfoAmazon(t *testing.T) {

 	r := newRedhat(config.ServerInfo{})
-	r.Family = "amazon"
+	r.Distro = config.Distro{Family: "redhat"}

 	issued, _ := time.Parse("2006-01-02", "2015-12-15")
 	updated, _ := time.Parse("2006-01-02", "2015-12-16")
@@ -601,7 +601,7 @@ Description : Package updates are available for Amazon Linux AMI that fix the

 func TestParseYumCheckUpdateLines(t *testing.T) {
 	r := newRedhat(config.ServerInfo{})
-	r.Family = "centos"
+	r.Distro = config.Distro{Family: "centos"}
 	stdout := `Loaded plugins: changelog, fastestmirror, keys, protect-packages, protectbase, security
 Loading mirror speeds from cached hostfile
 * base: mirror.fairway.ne.jp
@@ -709,7 +709,7 @@ bind-utils.x86_64                       30:9.3.6-25.P1.el5_11.8          updates

 func TestParseYumCheckUpdateLinesAmazon(t *testing.T) {
 	r := newRedhat(config.ServerInfo{})
-	r.Family = "amazon"
+	r.Distro = config.Distro{Family: "amazon"}
 	stdout := `Loaded plugins: priorities, update-motd, upgrade-helper
 34 package(s) needed for security, out of 71 available

@@ -1110,8 +1110,10 @@ func TestGetChangelogCVELines(t *testing.T) {
 	}

 	r := newRedhat(config.ServerInfo{})
-	r.Family = "centos"
-	r.Release = "6.7"
+	r.Distro = config.Distro{
+		Family:  "centos",
+		Release: "6.7",
+	}
 	for _, tt := range testsCentos6 {
 		rpm2changelog, err := r.parseAllChangelog(stdoutCentos6)
 		if err != nil {
@@ -1194,7 +1196,10 @@ func TestGetChangelogCVELines(t *testing.T) {
 		},
 	}

-	r.Release = "5.6"
+	r.Distro = config.Distro{
+		Family:  "centos",
+		Release: "5.6",
+	}
 	for _, tt := range testsCentos5 {
 		rpm2changelog, err := r.parseAllChangelog(stdoutCentos5)
 		if err != nil {
--- a/scan/serverapi.go
+++ b/scan/serverapi.go
@@ -1,3 +1,20 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package scan

 import (
@@ -5,6 +22,7 @@ import (
 	"time"

 	"github.com/Sirupsen/logrus"
+	"github.com/future-architect/vuls/cache"
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 	cve "github.com/kotakanbe/go-cve-dictionary/models"
@@ -20,8 +38,9 @@ type osTypeInterface interface {
 	setServerInfo(config.ServerInfo)
 	getServerInfo() config.ServerInfo

-	setDistributionInfo(string, string)
-	getDistributionInfo() string
+	setDistro(string, string)
+	getDistro() config.Distro
+	//  getFamily() string

 	checkIfSudoNoPasswd() error
 	detectPlatform() error
@@ -188,7 +207,7 @@ func detectServerOSes() (sshAbleOses []osTypeInterface) {
 				Log.Infof("(%d/%d) Detected: %s: %s",
 					i+1, len(config.Conf.Servers),
 					res.getServerInfo().ServerName,
-					res.getDistributionInfo())
+					res.getDistro())
 			}
 		case <-timeout:
 			msg := "Timed out while detecting servers"
@@ -248,7 +267,7 @@ func detectContainerOSes() (actives []osTypeInterface) {
 				}
 				oses = append(oses, res...)
 				Log.Infof("Detected: %s@%s: %s",
-					sinfo.Container.Name, sinfo.ServerName, osi.getDistributionInfo())
+					sinfo.Container.Name, sinfo.ServerName, osi.getDistro())
 			}
 		case <-timeout:
 			msg := "Timed out while detecting containers"
@@ -417,6 +436,13 @@ func Scan() []error {
 		return errs
 	}

+	if err := setupCangelogCache(); err != nil {
+		return []error{err}
+	}
+	if cache.DB != nil {
+		defer cache.DB.Close()
+	}
+
 	Log.Info("Scanning vulnerable OS packages...")
 	if errs := scanPackages(); errs != nil {
 		return errs
@@ -429,6 +455,23 @@ func Scan() []error {
 	return nil
 }

+func setupCangelogCache() error {
+	needToSetupCache := false
+	for _, s := range servers {
+		switch s.getDistro().Family {
+		case "ubuntu", "debian":
+			needToSetupCache = true
+			break
+		}
+	}
+	if needToSetupCache {
+		if err := cache.SetupBolt(config.Conf.CacheDBPath, Log); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func checkRequiredPackagesInstalled() []error {
 	timeoutSec := 30 * 60
 	return parallelSSHExec(func(o osTypeInterface) error {
--- a/scan/sshutil.go
+++ b/scan/sshutil.go
@@ -302,7 +302,7 @@ func decolateCmd(c conf.ServerInfo, cmd string, sudo bool) string {
 		cmd = strings.Replace(cmd, "|", "| sudo ", -1)
 	}

-	if c.Family != "FreeBSD" {
+	if c.Distro.Family != "FreeBSD" {
 		// set pipefail option. Bash only
 		// http://unix.stackexchange.com/questions/14270/get-exit-status-of-process-thats-piped-to-another
 		cmd = fmt.Sprintf("set -o pipefail; %s", cmd)