From 42a6004c7d1189097a66f2a652d50c983e55f9b5 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Sat, 28 Jan 2017 03:57:21 +0900 Subject: [PATCH] Fix the changelog cache logic for ubuntu/debian --- cache/bolt.go | 22 ++++++++++++-- cache/bolt_test.go | 5 +++- cache/db.go | 10 +++++-- scan/debian.go | 70 ++++++++++++++++++++++++++------------------- scan/debian_test.go | 8 +++--- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/cache/bolt.go b/cache/bolt.go index b085c71d..9edaf3b4 100644 --- a/cache/bolt.go +++ b/cache/bolt.go @@ -20,6 +20,7 @@ package cache import ( "encoding/json" "fmt" + "time" "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" @@ -92,6 +93,23 @@ func (b Bolt) GetMeta(serverName string) (meta Meta, found bool, err error) { return } +// RefreshMeta gets a Meta Information os the servername to boltdb. +func (b Bolt) RefreshMeta(meta Meta) error { + meta.CreatedAt = time.Now() + jsonBytes, err := json.Marshal(meta) + if err != nil { + return fmt.Errorf("Failed to marshal to JSON: %s", err) + } + return b.db.Update(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(metabucket)) + if err := bkt.Put([]byte(meta.Name), jsonBytes); err != nil { + return err + } + b.Log.Debugf("Refreshed Meta: %s", meta.Name) + return nil + }) +} + // EnsureBuckets puts a Meta information and create a buket that holds changelogs. func (b Bolt) EnsureBuckets(meta Meta) error { jsonBytes, err := json.Marshal(meta) @@ -123,12 +141,12 @@ func (b Bolt) EnsureBuckets(meta Meta) error { }) } -// PrettyPrint is for debuging +// PrettyPrint is for debug func (b Bolt) PrettyPrint(meta Meta) error { return b.db.View(func(tx *bolt.Tx) error { bkt := tx.Bucket([]byte(metabucket)) v := bkt.Get([]byte(meta.Name)) - b.Log.Debugf("key:%s, value:%s", meta.Name, v) + b.Log.Debugf("Meta: key:%s, value:%s", meta.Name, v) bkt = tx.Bucket([]byte(meta.Name)) c := bkt.Cursor() diff --git a/cache/bolt_test.go b/cache/bolt_test.go index a8dad918..ea2a31b4 100644 --- a/cache/bolt_test.go +++ b/cache/bolt_test.go @@ -90,9 +90,12 @@ func TestEnsureBuckets(t *testing.T) { if !found { t.Errorf("Not Found in meta") } - if !reflect.DeepEqual(meta, m) { + if meta.Name != m.Name || meta.Distro != m.Distro { t.Errorf("expected %v, actual %v", meta, m) } + if !reflect.DeepEqual(meta.Packs, m.Packs) { + t.Errorf("expected %v, actual %v", meta.Packs, m.Packs) + } if err := DB.Close(); err != nil { t.Errorf("Failed to close bolt: %s", err) } diff --git a/cache/db.go b/cache/db.go index 607d7bf7..30299390 100644 --- a/cache/db.go +++ b/cache/db.go @@ -18,6 +18,8 @@ along with this program. If not, see . package cache import ( + "time" + "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" ) @@ -31,6 +33,7 @@ const metabucket = "changelog-meta" type Cache interface { Close() error GetMeta(string) (Meta, bool, error) + RefreshMeta(Meta) error EnsureBuckets(Meta) error PrettyPrint(Meta) error GetChangelog(string, string) (string, error) @@ -40,9 +43,10 @@ type Cache interface { // Meta holds a server name, distro information of the scanned server and // package information that was collected at the last scan. type Meta struct { - Name string - Distro config.Distro - Packs []models.PackageInfo + Name string + Distro config.Distro + Packs []models.PackageInfo + CreatedAt time.Time } // FindPack search a PackageInfo diff --git a/scan/debian.go b/scan/debian.go index 9dd2dce2..77b3d6c7 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -277,13 +277,15 @@ func (o *debian) scanUnsecurePackages(installed []models.PackageInfo) ([]models. Distro: o.getServerInfo().Distro, Packs: upgradablePacks, } + o.log.Debugf("Ensure changelog cache: %s", current.Name) - if err := o.ensureChangelogCache(current); err != nil { + var meta *cache.Meta + if meta, err = o.ensureChangelogCache(current); err != nil { return nil, err } // Collect CVE information of upgradable packages - vulnInfos, err := o.scanVulnInfos(upgradablePacks) + vulnInfos, err := o.scanVulnInfos(upgradablePacks, meta) if err != nil { return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err) } @@ -291,35 +293,38 @@ func (o *debian) scanUnsecurePackages(installed []models.PackageInfo) ([]models. return vulnInfos, nil } -func (o *debian) ensureChangelogCache(current cache.Meta) error { +func (o *debian) ensureChangelogCache(current cache.Meta) (*cache.Meta, error) { // Search from cache - old, found, err := cache.DB.GetMeta(current.Name) + cached, found, err := cache.DB.GetMeta(current.Name) if err != nil { - return fmt.Errorf("Failed to get meta. err: %s", err) + return nil, fmt.Errorf("Failed to get meta. err: %s", err) } + if !found { o.log.Debugf("Not found in meta: %s", current.Name) err = cache.DB.EnsureBuckets(current) if err != nil { - return fmt.Errorf("Failed to ensure buckets. err: %s", err) - } - } else { - if current.Distro.Family != old.Distro.Family || - current.Distro.Release != old.Distro.Release { - o.log.Debugf("Need to refesh meta: %s", current.Name) - err = cache.DB.EnsureBuckets(current) - if err != nil { - return fmt.Errorf("Failed to ensure buckets. err: %s", err) - } - } else { - o.log.Debugf("Reuse meta: %s", current.Name) + return nil, fmt.Errorf("Failed to ensure buckets. err: %s", err) } + return ¤t, nil } + if current.Distro.Family != cached.Distro.Family || + current.Distro.Release != cached.Distro.Release { + o.log.Debugf("Need to refesh meta: %s", current.Name) + err = cache.DB.EnsureBuckets(current) + if err != nil { + return nil, fmt.Errorf("Failed to ensure buckets. err: %s", err) + } + return ¤t, nil + + } + + o.log.Debugf("Reuse meta: %s", current.Name) if config.Conf.Debug { cache.DB.PrettyPrint(current) } - return nil + return &cached, nil } func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []models.PackageInfo, err error) { @@ -401,13 +406,7 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er return } -func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo) (models.VulnInfos, error) { - meta := cache.Meta{ - Name: o.getServerInfo().GetServerName(), - Distro: o.getServerInfo().Distro, - Packs: upgradablePacks, - } - +func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache.Meta) (models.VulnInfos, error) { type strarray []string resChan := make(chan struct { models.PackageInfo @@ -445,6 +444,7 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo) (models.Vul // if the changelog is not in cache or failed to get from local cache, // get the changelog of the package via internet. + // After that, store it in the cache. if cveIDs, err := o.scanPackageCveIDs(p); err != nil { errChan <- err } else { @@ -493,15 +493,26 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo) (models.Vul Packages: v, }) } + + // Update meta package information of changelog cache to the latest one. + meta.Packs = upgradablePacks + if err := cache.DB.RefreshMeta(*meta); err != nil { + return nil, err + } + return vinfos, nil } -func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) string { +func (o *debian) getChangelogCache(meta *cache.Meta, pack models.PackageInfo) string { cachedPack, found := meta.FindPack(pack.Name) if !found { + o.log.Debugf("Not found: %s", pack.Name) return "" } + if cachedPack.NewVersion != pack.NewVersion { + o.log.Debugf("Expired: %s, cache: %s, new: %s", + pack.Name, cachedPack.NewVersion, pack.NewVersion) return "" } changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name) @@ -511,11 +522,12 @@ func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) str return "" } if len(changelog) == 0 { + o.log.Debugf("Empty string: %s", pack.Name) return "" } - o.log.Debugf("Cache hit: %s, len: %d, %s...", - meta.Name, len(changelog), util.Truncate(changelog, 30)) + o.log.Debugf("Hit: %s, %s, cache: %s, new: %s len: %d, %s...", + meta.Name, pack.Name, cachedPack.NewVersion, pack.NewVersion, len(changelog), util.Truncate(changelog, 30)) return changelog } @@ -583,7 +595,7 @@ func (o *debian) parseChangelog(changelog string, lines := strings.Split(changelog, "\n") for _, line := range lines { if matche := stopRe.MatchString(line); matche { - o.log.Debugf("Found the stop line. line: %s", line) + // o.log.Debugf("Found the stop line: %s", line) stopLineFound = true break } else if matches := cveRe.FindAllString(line, -1); 0 < len(matches) { diff --git a/scan/debian_test.go b/scan/debian_test.go index b49f528c..946be3d0 100644 --- a/scan/debian_test.go +++ b/scan/debian_test.go @@ -545,7 +545,7 @@ func TestGetChangelogCache(t *testing.T) { } d := newDebian(config.ServerInfo{}) - actual := d.getChangelogCache(meta, pack) + actual := d.getChangelogCache(&meta, pack) if actual != "" { t.Errorf("Failed to get empty stirng from cache:") } @@ -555,21 +555,21 @@ func TestGetChangelogCache(t *testing.T) { t.Errorf("Failed to put changelog: %s", err) } - actual = d.getChangelogCache(meta, pack) + actual = d.getChangelogCache(&meta, pack) if actual != clog { t.Errorf("Failed to get changelog from cache: %s", actual) } // increment a version of the pack pack.NewVersion = "1.0.2" - actual = d.getChangelogCache(meta, pack) + actual = d.getChangelogCache(&meta, pack) if actual != "" { t.Errorf("The changelog is not invalidated: %s", actual) } // change a name of the pack pack.Name = "bash" - actual = d.getChangelogCache(meta, pack) + actual = d.getChangelogCache(&meta, pack) if actual != "" { t.Errorf("The changelog is not invalidated: %s", actual) }