Fix the changelog cache logic for ubuntu/debian

cache/bolt.go

@@ -20,6 +20,7 @@ package cache
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/boltdb/bolt"
@@ -92,6 +93,23 @@ func (b Bolt) GetMeta(serverName string) (meta Meta, found bool, err error) {
 	return
 }
 
+// RefreshMeta gets a Meta Information os the servername to boltdb.
+func (b Bolt) RefreshMeta(meta Meta) error {
+	meta.CreatedAt = time.Now()
+	jsonBytes, err := json.Marshal(meta)
+	if err != nil {
+		return fmt.Errorf("Failed to marshal to JSON: %s", err)
+	}
+	return b.db.Update(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket([]byte(metabucket))
+		if err := bkt.Put([]byte(meta.Name), jsonBytes); err != nil {
+			return err
+		}
+		b.Log.Debugf("Refreshed Meta: %s", meta.Name)
+		return nil
+	})
+}
+
 // EnsureBuckets puts a Meta information and create a buket that holds changelogs.
 func (b Bolt) EnsureBuckets(meta Meta) error {
 	jsonBytes, err := json.Marshal(meta)
@@ -123,12 +141,12 @@ func (b Bolt) EnsureBuckets(meta Meta) error {
 	})
 }
 
-// PrettyPrint is for debuging
+// PrettyPrint is for debug
 func (b Bolt) PrettyPrint(meta Meta) error {
 	return b.db.View(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket([]byte(metabucket))
 		v := bkt.Get([]byte(meta.Name))
-		b.Log.Debugf("key:%s, value:%s", meta.Name, v)
+		b.Log.Debugf("Meta: key:%s, value:%s", meta.Name, v)
 
 		bkt = tx.Bucket([]byte(meta.Name))
 		c := bkt.Cursor()

cache/bolt_test.go

@@ -90,9 +90,12 @@ func TestEnsureBuckets(t *testing.T) {
 	if !found {
 		t.Errorf("Not Found in meta")
 	}
-	if !reflect.DeepEqual(meta, m) {
+	if meta.Name != m.Name || meta.Distro != m.Distro {
 		t.Errorf("expected %v, actual %v", meta, m)
 	}
+	if !reflect.DeepEqual(meta.Packs, m.Packs) {
+		t.Errorf("expected %v, actual %v", meta.Packs, m.Packs)
+	}
 	if err := DB.Close(); err != nil {
 		t.Errorf("Failed to close bolt: %s", err)
 	}

cache/db.go

@@ -18,6 +18,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package cache
 
 import (
+	"time"
+
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 )
@@ -31,6 +33,7 @@ const metabucket = "changelog-meta"
 type Cache interface {
 	Close() error
 	GetMeta(string) (Meta, bool, error)
+	RefreshMeta(Meta) error
 	EnsureBuckets(Meta) error
 	PrettyPrint(Meta) error
 	GetChangelog(string, string) (string, error)
@@ -40,9 +43,10 @@ type Cache interface {
 // Meta holds a server name, distro information of the scanned server and
 // package information that was collected at the last scan.
 type Meta struct {
-	Name   string
-	Distro config.Distro
-	Packs  []models.PackageInfo
+	Name      string
+	Distro    config.Distro
+	Packs     []models.PackageInfo
+	CreatedAt time.Time
 }
 
 // FindPack search a PackageInfo

scan/debian.go

@@ -277,13 +277,15 @@ func (o *debian) scanUnsecurePackages(installed []models.PackageInfo) ([]models.
 		Distro: o.getServerInfo().Distro,
 		Packs:  upgradablePacks,
 	}
+
 	o.log.Debugf("Ensure changelog cache: %s", current.Name)
-	if err := o.ensureChangelogCache(current); err != nil {
+	var meta *cache.Meta
+	if meta, err = o.ensureChangelogCache(current); err != nil {
 		return nil, err
 	}
 
 	// Collect CVE information of upgradable packages
-	vulnInfos, err := o.scanVulnInfos(upgradablePacks)
+	vulnInfos, err := o.scanVulnInfos(upgradablePacks, meta)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err)
 	}
@@ -291,35 +293,38 @@ func (o *debian) scanUnsecurePackages(installed []models.PackageInfo) ([]models.
 	return vulnInfos, nil
 }
 
-func (o *debian) ensureChangelogCache(current cache.Meta) error {
+func (o *debian) ensureChangelogCache(current cache.Meta) (*cache.Meta, error) {
 	// Search from cache
-	old, found, err := cache.DB.GetMeta(current.Name)
+	cached, found, err := cache.DB.GetMeta(current.Name)
 	if err != nil {
-		return fmt.Errorf("Failed to get meta. err: %s", err)
+		return nil, fmt.Errorf("Failed to get meta. err: %s", err)
 	}
+
 	if !found {
 		o.log.Debugf("Not found in meta: %s", current.Name)
 		err = cache.DB.EnsureBuckets(current)
 		if err != nil {
-			return fmt.Errorf("Failed to ensure buckets. err: %s", err)
-		}
-	} else {
-		if current.Distro.Family != old.Distro.Family ||
-			current.Distro.Release != old.Distro.Release {
-			o.log.Debugf("Need to refesh meta: %s", current.Name)
-			err = cache.DB.EnsureBuckets(current)
-			if err != nil {
-				return fmt.Errorf("Failed to ensure buckets. err: %s", err)
-			}
-		} else {
-			o.log.Debugf("Reuse meta: %s", current.Name)
+			return nil, fmt.Errorf("Failed to ensure buckets. err: %s", err)
 		}
+		return &current, nil
 	}
 
+	if current.Distro.Family != cached.Distro.Family ||
+		current.Distro.Release != cached.Distro.Release {
+		o.log.Debugf("Need to refesh meta: %s", current.Name)
+		err = cache.DB.EnsureBuckets(current)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to ensure buckets. err: %s", err)
+		}
+		return &current, nil
+
+	}
+
+	o.log.Debugf("Reuse meta: %s", current.Name)
 	if config.Conf.Debug {
 		cache.DB.PrettyPrint(current)
 	}
-	return nil
+	return &cached, nil
 }
 
 func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []models.PackageInfo, err error) {
@@ -401,13 +406,7 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er
 	return
 }
 
-func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo) (models.VulnInfos, error) {
-	meta := cache.Meta{
-		Name:   o.getServerInfo().GetServerName(),
-		Distro: o.getServerInfo().Distro,
-		Packs:  upgradablePacks,
-	}
-
+func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache.Meta) (models.VulnInfos, error) {
 	type strarray []string
 	resChan := make(chan struct {
 		models.PackageInfo
@@ -445,6 +444,7 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo) (models.Vul
 
 					// if the changelog is not in cache or failed to get from local cache,
 					// get the changelog of the package via internet.
+					// After that, store it in the cache.
 					if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
 						errChan <- err
 					} else {
@@ -493,15 +493,26 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo) (models.Vul
 			Packages: v,
 		})
 	}
+
+	// Update meta package information of changelog cache to the latest one.
+	meta.Packs = upgradablePacks
+	if err := cache.DB.RefreshMeta(*meta); err != nil {
+		return nil, err
+	}
+
 	return vinfos, nil
 }
 
-func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) string {
+func (o *debian) getChangelogCache(meta *cache.Meta, pack models.PackageInfo) string {
 	cachedPack, found := meta.FindPack(pack.Name)
 	if !found {
+		o.log.Debugf("Not found: %s", pack.Name)
 		return ""
 	}
+
 	if cachedPack.NewVersion != pack.NewVersion {
+		o.log.Debugf("Expired: %s, cache: %s, new: %s",
+			pack.Name, cachedPack.NewVersion, pack.NewVersion)
 		return ""
 	}
 	changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name)
@@ -511,11 +522,12 @@ func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) str
 		return ""
 	}
 	if len(changelog) == 0 {
+		o.log.Debugf("Empty string: %s", pack.Name)
 		return ""
 	}
 
-	o.log.Debugf("Cache hit: %s, len: %d, %s...",
-		meta.Name, len(changelog), util.Truncate(changelog, 30))
+	o.log.Debugf("Hit: %s, %s, cache: %s, new: %s len: %d, %s...",
+		meta.Name, pack.Name, cachedPack.NewVersion, pack.NewVersion, len(changelog), util.Truncate(changelog, 30))
 	return changelog
 }
 
@@ -583,7 +595,7 @@ func (o *debian) parseChangelog(changelog string,
 	lines := strings.Split(changelog, "\n")
 	for _, line := range lines {
 		if matche := stopRe.MatchString(line); matche {
-			o.log.Debugf("Found the stop line. line: %s", line)
+			//  o.log.Debugf("Found the stop line: %s", line)
 			stopLineFound = true
 			break
 		} else if matches := cveRe.FindAllString(line, -1); 0 < len(matches) {

scan/debian_test.go

@@ -545,7 +545,7 @@ func TestGetChangelogCache(t *testing.T) {
 	}
 
 	d := newDebian(config.ServerInfo{})
-	actual := d.getChangelogCache(meta, pack)
+	actual := d.getChangelogCache(&meta, pack)
 	if actual != "" {
 		t.Errorf("Failed to get empty stirng from cache:")
 	}
@@ -555,21 +555,21 @@ func TestGetChangelogCache(t *testing.T) {
 		t.Errorf("Failed to put changelog: %s", err)
 	}
 
-	actual = d.getChangelogCache(meta, pack)
+	actual = d.getChangelogCache(&meta, pack)
 	if actual != clog {
 		t.Errorf("Failed to get changelog from cache: %s", actual)
 	}
 
 	// increment a version of the pack
 	pack.NewVersion = "1.0.2"
-	actual = d.getChangelogCache(meta, pack)
+	actual = d.getChangelogCache(&meta, pack)
 	if actual != "" {
 		t.Errorf("The changelog is not invalidated: %s", actual)
 	}
 
 	// change a name of the pack
 	pack.Name = "bash"
-	actual = d.getChangelogCache(meta, pack)
+	actual = d.getChangelogCache(&meta, pack)
 	if actual != "" {
 		t.Errorf("The changelog is not invalidated: %s", actual)
 	}