From a9ebac3818128c3c806383114b1e7178baad3e3f Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Tue, 18 Jul 2017 15:54:25 +0900 Subject: [PATCH] nosudo on CentOS and Fetch Changelogs on Amazon, RHEL (#448) * Use repoquery for no sudo and avoid unintended line feed of yum or rpm. #444 * Change data type of enablerepo in config.toml. string to array * Fetch yum changelogs at once then grep CVE-IDs * Fix changelog parse logic and Update Gopkg --- .travis.yml | 1 - Gopkg.lock | 61 +-- Gopkg.toml | 74 +++- README.md | 12 +- commands/scan.go | 1 + config/config.go | 2 +- config/tomlloader.go | 3 +- models/models.go | 2 +- models/packages.go | 16 +- models/scanresults.go | 2 +- models/vulninfos.go | 10 +- report/azureblob.go | 2 +- report/tui.go | 50 ++- scan/debian.go | 56 +-- scan/redhat.go | 713 ++++++++++++++++------------------- scan/redhat_test.go | 855 ++++++++++++++++++++++-------------------- 16 files changed, 944 insertions(+), 916 deletions(-) diff --git a/.travis.yml b/.travis.yml index c4552c6f..2875f639 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.7 - 1.8 diff --git a/Gopkg.lock b/Gopkg.lock index 0587848c..bc967282 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,10 +2,10 @@ [[projects]] - branch = "master" - name = "github.com/Azure/azure-storage-go" - packages = ["."] - revision = "32cfbe17a139c17f84be16bdf8f9c45c840a046b" + name = "github.com/Azure/azure-sdk-for-go" + packages = ["storage"] + revision = "59c277f1b488b81b1a5f944212f25b69bea8ece3" + version = "v10.1.0-beta" [[projects]] name = "github.com/Azure/go-autorest" @@ -14,10 +14,10 @@ version = "v8.1.0" [[projects]] - branch = "master" name = "github.com/BurntSushi/toml" packages = ["."] - revision = "a368813c5e648fee92e5f6c30e3944ff9d5e8895" + revision = "b26d9c308763d68093482582cea63d69be07a0f0" + version = "v0.3.0" [[projects]] name = "github.com/asaskevich/govalidator" @@ -27,14 +27,15 @@ [[projects]] name = "github.com/aws/aws-sdk-go" - packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/request","aws/session","aws/signer/v4","private/endpoints","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","private/waiter","service/s3","service/sts"] - revision = "5b341290c488aa6bd76b335d819b4a68516ec3ab" + packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"] + revision = "6d7fc1a00fcae6bbb53550f4a0b98324fd7aa250" + version = "v1.10.12" [[projects]] name = "github.com/boltdb/bolt" packages = ["."] - revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9" - version = "v1.3.0" + revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" + version = "v1.3.1" [[projects]] name = "github.com/cenkalti/backoff" @@ -63,8 +64,8 @@ [[projects]] name = "github.com/go-redis/redis" packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto"] - revision = "e14976b254c5bc5f399dd0ae9314b1d02a176897" - version = "v6.5.0" + revision = "da63fe7def48e378caf9539abf64b9b1e37bc01e" + version = "v6.5.3" [[projects]] name = "github.com/go-sql-driver/mysql" @@ -109,10 +110,10 @@ version = "0.2.2" [[projects]] - branch = "master" name = "github.com/jroimartin/gocui" packages = ["."] - revision = "612b0b2987ec1a6af46d7008cef1efd4b3898346" + revision = "4e9ce9a8e26f2ef33dfe297dbdfca148733b6b9b" + version = "v0.3.0" [[projects]] branch = "master" @@ -128,9 +129,15 @@ [[projects]] branch = "master" + name = "github.com/knqyf263/go-rpm-version" + packages = ["."] + revision = "74609b86c936dff800c69ec89fcf4bc52d5f13a4" + +[[projects]] name = "github.com/kotakanbe/go-cve-dictionary" packages = ["config","db","jvn","log","models","nvd","util"] revision = "89e381b4e7e5a31097bbd5779cbb555f5bd3fe87" + version = "v0.1.1" [[projects]] name = "github.com/kotakanbe/go-pingscanner" @@ -153,14 +160,14 @@ [[projects]] name = "github.com/labstack/gommon" packages = ["color","log"] - revision = "1121fd3e243c202482226a7afe4dcd07ffc4139a" - version = "v0.2.1" + revision = "779b8a8b9850a97acba6a3fe20feb628c39e17c1" + version = "0.2.2" [[projects]] branch = "master" name = "github.com/lib/pq" packages = [".","hstore","oid"] - revision = "8837942c3e09574accbc5f150e2c5e057189cace" + revision = "dd1fe2071026ce53f36a39112e645b4d4f5793a4" [[projects]] name = "github.com/mattn/go-colorable" @@ -202,7 +209,7 @@ branch = "master" name = "github.com/nsf/termbox-go" packages = ["."] - revision = "72800b73ab9a3c78df350738298b0361354772ff" + revision = "4ed959e0540971545eddb8c75514973d670cf739" [[projects]] name = "github.com/parnurzeal/gorequest" @@ -232,7 +239,7 @@ branch = "master" name = "github.com/sirupsen/logrus" packages = ["."] - revision = "3d4380f53a34dcdc95f0c1db702615992b38d9a4" + revision = "5ff5dd844dfeb4e23e27528f79f1f845bc8bb78f" [[projects]] branch = "master" @@ -252,39 +259,33 @@ packages = ["oval"] revision = "003ac9af5fffac6c97ab1def025d2cb73e88469a" -[[projects]] - branch = "master" - name = "go4.org" - packages = ["syncutil"] - revision = "034d17a462f7b2dcd1a4a73553ec5357ff6e6c6e" - [[projects]] branch = "master" name = "golang.org/x/crypto" packages = ["curve25519","ed25519","ed25519/internal/edwards25519","ssh","ssh/agent","ssh/terminal"] - revision = "adbae1b6b6fb4b02448a0fc0dbbc9ba2b95b294d" + revision = "7f7c0c2d75ebb4e32a21396ce36e87b6dadc91c9" [[projects]] branch = "master" name = "golang.org/x/net" packages = ["context","idna","publicsuffix"] - revision = "455220fa52c866a8aa14ff5e8cc68cde16b8395e" + revision = "b3756b4b77d7b13260a0a2ec658753cf48922eac" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "90796e5a05ce440b41c768bd9af257005e470461" + revision = "4cd6d1a821c7175768725b55ca82f14683a29ea4" [[projects]] branch = "master" name = "golang.org/x/text" packages = ["internal/gen","internal/triegen","internal/ucd","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] - revision = "6353ef0f924300eea566d3438817aa4d3374817e" + revision = "836efe42bb4aa16aaa17b9c155d8813d336ed720" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "a6b387c74e75e1f971ee643c8904f6fd4e3dfdb7fa36119ab7bc28d9cfd66427" + inputs-digest = "269ff02f8e4540ba049a340068dac0ff4f0495df9f8eeb21d4a545ea5dedf2dd" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 9e0d6fb4..da91d653 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,23 +1,57 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + + [[constraint]] - branch = "master" - name = "github.com/Azure/azure-storage-go" - -[[constraint]] - branch = "master" name = "github.com/BurntSushi/toml" + version = "0.3.0" + +[[constraint]] + name = "github.com/asaskevich/govalidator" + version = "6.0.0" + +[[constraint]] + name = "github.com/boltdb/bolt" + version = "1.3.1" + +[[constraint]] + name = "github.com/cenkalti/backoff" + version = "1.0.0" [[constraint]] branch = "master" - name = "github.com/sirupsen/logrus" - -[[constraint]] - name = "github.com/aws/aws-sdk-go" - revision = "5b341290c488aa6bd76b335d819b4a68516ec3ab" + name = "github.com/google/subcommands" [[constraint]] branch = "master" + name = "github.com/gosuri/uitable" + +[[constraint]] + branch = "master" + name = "github.com/howeyc/gopass" + +[[constraint]] name = "github.com/jroimartin/gocui" + version = "0.3.0" [[constraint]] branch = "master" @@ -25,12 +59,28 @@ [[constraint]] branch = "master" - name = "github.com/kotakanbe/go-cve-dictionary" + name = "github.com/knqyf263/go-deb-version" [[constraint]] branch = "master" - name = "github.com/kotakanbe/goval-dictionary" + name = "github.com/knqyf263/go-rpm-version" + +[[constraint]] + name = "github.com/kotakanbe/go-pingscanner" + version = "0.1.0" [[constraint]] branch = "master" name = "github.com/kotakanbe/logrus-prefixed-formatter" + +[[constraint]] + name = "github.com/parnurzeal/gorequest" + version = "0.2.15" + +[[constraint]] + name = "github.com/rifflock/lfshook" + version = "1.7.0" + +[[constraint]] + branch = "master" + name = "github.com/sirupsen/logrus" diff --git a/README.md b/README.md index e9f66dc6..4dc7cf46 100644 --- a/README.md +++ b/README.md @@ -803,12 +803,12 @@ In order to scan, the following dependencies are required, so you need to instal |:-------------|-------------------:|:-------------| | Ubuntu | 12, 14, 16| - | | Debian | 7, 8| aptitude | -| CentOS | 6, 7| yum-plugin-changelog | -| Amazon | All | - | -| RHEL | 5 | yum-security | -| RHEL | 6, 7 | - | -| Oracle Linux | 5 | yum-security | -| Oracle Linux | 6, 7 | - | +| CentOS | 6, 7| yum-plugin-changelog, yum-utils | +| Amazon | All | - | TODO yum-utils?, yum-plugin-changelog +| RHEL | 5 | yum-security | TODO yum-utils? +| RHEL | 6, 7 | - | TODO yum-utils? +| Oracle Linux | 5 | yum-security | TODO yum-utils? +| Oracle Linux | 6, 7 | - |TODO yum-utils? | FreeBSD | 10 | - | | Raspbian | Wheezy, Jessie | - | diff --git a/commands/scan.go b/commands/scan.go index 5359be60..0ce52481 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -67,6 +67,7 @@ func (*ScanCmd) Usage() string { [-cachedb-path=/path/to/cache.db] [-ssh-native-insecure] [-containers-only] + [-package-list-only] [-skip-broken] [-http-proxy=http://192.168.0.1:8080] [-ask-key-password] diff --git a/config/config.go b/config/config.go index 2ad9c073..edbaa433 100644 --- a/config/config.go +++ b/config/config.go @@ -418,7 +418,7 @@ type ServerInfo struct { Optional [][]interface{} // For CentOS, RHEL, Amazon - Enablerepo string + Enablerepo []string // used internal LogMsgAnsiColor string // DebugLog Color diff --git a/config/tomlloader.go b/config/tomlloader.go index 7030dd0b..1cadd8cc 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -20,7 +20,6 @@ package config import ( "fmt" "os" - "strings" "github.com/BurntSushi/toml" "github.com/future-architect/vuls/contrib/owasp-dependency-check/parser" @@ -164,7 +163,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { s.Enablerepo = d.Enablerepo } if len(s.Enablerepo) != 0 { - for _, repo := range strings.Split(s.Enablerepo, ",") { + for _, repo := range s.Enablerepo { switch repo { case "base", "updates": // nop diff --git a/models/models.go b/models/models.go index bd7ab57f..96850f24 100644 --- a/models/models.go +++ b/models/models.go @@ -18,4 +18,4 @@ along with this program. If not, see . package models // JSONVersion is JSON Version -const JSONVersion = "0.3.0" +const JSONVersion = 2 diff --git a/models/packages.go b/models/packages.go index 19ed3c5f..827ca103 100644 --- a/models/packages.go +++ b/models/packages.go @@ -42,6 +42,7 @@ func (ps Packages) MergeNewVersion(as Packages) { if pack, ok := ps[a.Name]; ok { pack.NewVersion = a.NewVersion pack.NewRelease = a.NewRelease + pack.Repository = a.Repository ps[a.Name] = pack } } @@ -79,6 +80,16 @@ func (ps Packages) FormatUpdatablePacksSummary() string { return fmt.Sprintf("%d updatable packages", nUpdatable) } +// FindOne search a element by name-newver-newrel-arch +func (ps Packages) FindOne(f func(Package) bool) (string, Package, bool) { + for key, p := range ps { + if f(p) { + return key, p, true + } + } + return "", Package{}, false +} + // Package has installed packages. type Package struct { Name string @@ -86,6 +97,7 @@ type Package struct { Release string NewVersion string NewRelease string + Arch string Repository string Changelog Changelog NotFixedYet bool // Ubuntu OVAL Only @@ -145,8 +157,8 @@ func (p Package) FormatChangelog() string { } // Changelog has contents of changelog and how to get it. -// Method: modesl.detectionMethodStr +// Method: models.detectionMethodStr type Changelog struct { Contents string - Method string + Method DetectionMethod } diff --git a/models/scanresults.go b/models/scanresults.go index aa538249..284724fb 100644 --- a/models/scanresults.go +++ b/models/scanresults.go @@ -32,7 +32,7 @@ type ScanResults []ScanResult // ScanResult has the result of scanned CVE information. type ScanResult struct { ScannedAt time.Time - JSONVersion string + JSONVersion int Lang string ServerName string // TOML Section key Family string diff --git a/models/vulninfos.go b/models/vulninfos.go index 2c31e8b7..f0e92b82 100644 --- a/models/vulninfos.go +++ b/models/vulninfos.go @@ -26,7 +26,8 @@ import ( "github.com/future-architect/vuls/config" ) -// VulnInfos is VulnInfo list, getter/setter, sortable methods. +// VulnInfos has a map of VulnInfo +// Key: CveID type VulnInfos map[string]VulnInfo // Find elements that matches the function passed in argument @@ -198,13 +199,18 @@ type DistroAdvisory struct { // Score: 0 - 100 type Confidence struct { Score int - DetectionMethod string + DetectionMethod DetectionMethod } func (c Confidence) String() string { return fmt.Sprintf("%d / %s", c.Score, c.DetectionMethod) } +// DetectionMethod indicates +// - How to detect the CveID +// - How to get the changelog difference between installed and candidate version +type DetectionMethod string + const ( // CpeNameMatchStr is a String representation of CpeNameMatch CpeNameMatchStr = "CpeNameMatch" diff --git a/report/azureblob.go b/report/azureblob.go index 2220e9c1..f065accd 100644 --- a/report/azureblob.go +++ b/report/azureblob.go @@ -24,7 +24,7 @@ import ( "fmt" "time" - "github.com/Azure/azure-storage-go" + storage "github.com/Azure/azure-sdk-for-go/storage" c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" diff --git a/report/tui.go b/report/tui.go index 83b786f2..9f762f15 100644 --- a/report/tui.go +++ b/report/tui.go @@ -45,14 +45,12 @@ var currentDetailLimitY int func RunTui(results models.ScanResults) subcommands.ExitStatus { scanResults = results - g, err := gocui.NewGui(gocui.OutputNormal) - if err != nil { - log.Errorf("%s", err) - return subcommands.ExitFailure - } + // g, err := gocui.NewGui(gocui.OutputNormal) + g := gocui.NewGui() defer g.Close() - g.SetManagerFunc(layout) + g.SetLayout(layout) + // g.SetManagerFunc(layout) if err := keybindings(g); err != nil { log.Errorf("%s", err) return subcommands.ExitFailure @@ -177,19 +175,19 @@ func nextView(g *gocui.Gui, v *gocui.View) error { var err error if v == nil { - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") } switch v.Name() { case "side": - _, err = g.SetCurrentView("summary") + err = g.SetCurrentView("summary") case "summary": - _, err = g.SetCurrentView("detail") + err = g.SetCurrentView("detail") case "detail": - _, err = g.SetCurrentView("changelog") + err = g.SetCurrentView("changelog") case "changelog": - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") default: - _, err = g.SetCurrentView("summary") + err = g.SetCurrentView("summary") } return err } @@ -198,19 +196,19 @@ func previousView(g *gocui.Gui, v *gocui.View) error { var err error if v == nil { - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") } switch v.Name() { case "side": - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") case "summary": - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") case "detail": - _, err = g.SetCurrentView("summary") + err = g.SetCurrentView("summary") case "changelog": - _, err = g.SetCurrentView("detail") + err = g.SetCurrentView("detail") default: - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") } return err } @@ -393,7 +391,7 @@ func cursorPageUp(g *gocui.Gui, v *gocui.View) error { func previousSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if _, err := g.SetCurrentView("summary"); err != nil { + if err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -401,7 +399,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if _, err := g.SetCurrentView("detail"); err != nil { + if err := g.SetCurrentView("detail"); err != nil { return err } } @@ -411,7 +409,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { func nextSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if _, err := g.SetCurrentView("summary"); err != nil { + if err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -419,7 +417,7 @@ func nextSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if _, err := g.SetCurrentView("detail"); err != nil { + if err := g.SetCurrentView("detail"); err != nil { return err } } @@ -502,7 +500,7 @@ func getLine(g *gocui.Gui, v *gocui.View) error { return err } fmt.Fprintln(v, l) - if _, err := g.SetCurrentView("msg"); err != nil { + if err := g.SetCurrentView("msg"); err != nil { return err } } @@ -525,7 +523,7 @@ func showMsg(g *gocui.Gui, v *gocui.View) error { return err } fmt.Fprintln(v, l) - if _, err := g.SetCurrentView("msg"); err != nil { + if err := g.SetCurrentView("msg"); err != nil { return err } } @@ -536,7 +534,7 @@ func delMsg(g *gocui.Gui, v *gocui.View) error { if err := g.DeleteView("msg"); err != nil { return err } - if _, err := g.SetCurrentView("summary"); err != nil { + if err := g.SetCurrentView("summary"); err != nil { return err } return nil @@ -592,7 +590,7 @@ func setSideLayout(g *gocui.Gui) error { } currentScanResult = scanResults[0] vinfos = scanResults[0].ScannedCves.ToSortedSlice() - if _, err := g.SetCurrentView("side"); err != nil { + if err := g.SetCurrentView("side"); err != nil { return err } } diff --git a/scan/debian.go b/scan/debian.go index 17559ded..a7ecffb3 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -167,7 +167,7 @@ func (o *debian) checkDependencies() error { } func (o *debian) scanPackages() error { - installed, upgradable, err := o.scanInstalledPackages() + installed, updatable, err := o.scanInstalledPackages() if err != nil { o.log.Errorf("Failed to scan installed packages") return err @@ -178,7 +178,7 @@ func (o *debian) scanPackages() error { return nil } - unsecure, err := o.scanUnsecurePackages(upgradable) + unsecure, err := o.scanUnsecurePackages(updatable) if err != nil { o.log.Errorf("Failed to scan vulnerable packages") return err @@ -189,7 +189,7 @@ func (o *debian) scanPackages() error { func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, error) { installed := models.Packages{} - upgradable := models.Packages{} + updatable := models.Packages{} r := o.exec("dpkg-query -W", noSudo) if !r.isSuccess() { @@ -214,27 +214,27 @@ func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, erro } } - upgradableNames, err := o.GetUpgradablePackNames() + updatableNames, err := o.getUpdatablePackNames() if err != nil { return nil, nil, err } - for _, name := range upgradableNames { + for _, name := range updatableNames { for _, pack := range installed { if pack.Name == name { - upgradable[name] = pack + updatable[name] = pack break } } } // Fill the candidate versions of upgradable packages - err = o.fillCandidateVersion(upgradable) + err = o.fillCandidateVersion(updatable) if err != nil { return nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err) } - installed.MergeNewVersion(upgradable) + installed.MergeNewVersion(updatable) - return installed, upgradable, nil + return installed, updatable, nil } var packageLinePattern = regexp.MustCompile(`^([^\t']+)\t(.+)$`) @@ -263,14 +263,14 @@ func (o *debian) aptGetUpdate() error { return nil } -func (o *debian) scanUnsecurePackages(upgradable models.Packages) (models.VulnInfos, error) { +func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) { o.aptGetUpdate() // Setup changelog cache current := cache.Meta{ Name: o.getServerInfo().GetServerName(), Distro: o.getServerInfo().Distro, - Packs: upgradable, + Packs: updatable, } o.log.Debugf("Ensure changelog cache: %s", current.Name) @@ -280,7 +280,7 @@ func (o *debian) scanUnsecurePackages(upgradable models.Packages) (models.VulnIn } // Collect CVE information of upgradable packages - vulnInfos, err := o.scanVulnInfos(upgradable, meta) + vulnInfos, err := o.scanVulnInfos(updatable, meta) if err != nil { return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err) } @@ -349,7 +349,7 @@ func (o *debian) fillCandidateVersion(packages models.Packages) (err error) { return } -func (o *debian) GetUpgradablePackNames() (packNames []string, err error) { +func (o *debian) getUpdatablePackNames() (packNames []string, err error) { cmd := util.PrependProxyEnv("LANGUAGE=en_US.UTF-8 apt-get upgrade --dry-run") r := o.exec(cmd, noSudo) if r.isSuccess(0, 1) { @@ -360,7 +360,7 @@ func (o *debian) GetUpgradablePackNames() (packNames []string, err error) { cmd, r.ExitStatus, r.Stdout, r.Stderr) } -func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, err error) { +func (o *debian) parseAptGetUpgrade(stdout string) (updatableNames []string, err error) { startRe := regexp.MustCompile(`The following packages will be upgraded:`) stopRe := regexp.MustCompile(`^(\d+) upgraded.*`) startLineFound, stopLineFound := false, false @@ -375,21 +375,21 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er } result := stopRe.FindStringSubmatch(line) if len(result) == 2 { - numUpgradablePacks, err := strconv.Atoi(result[1]) + nUpdatable, err := strconv.Atoi(result[1]) if err != nil { return nil, fmt.Errorf( "Failed to scan upgradable packages number. line: %s", line) } - if numUpgradablePacks != len(upgradableNames) { + if nUpdatable != len(updatableNames) { return nil, fmt.Errorf( "Failed to scan upgradable packages, expected: %s, detected: %d", - result[1], len(upgradableNames)) + result[1], len(updatableNames)) } stopLineFound = true o.log.Debugf("Found the stop line. line: %s", line) break } - upgradableNames = append(upgradableNames, strings.Fields(line)...) + updatableNames = append(updatableNames, strings.Fields(line)...) } if !startLineFound { // no upgrades @@ -410,20 +410,20 @@ type DetectedCveID struct { Confidence models.Confidence } -func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) { +func (o *debian) scanVulnInfos(updatablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) { type response struct { pack *models.Package DetectedCveIDs []DetectedCveID } - resChan := make(chan response, len(upgradablePacks)) - errChan := make(chan error, len(upgradablePacks)) - reqChan := make(chan models.Package, len(upgradablePacks)) + resChan := make(chan response, len(updatablePacks)) + errChan := make(chan error, len(updatablePacks)) + reqChan := make(chan models.Package, len(updatablePacks)) defer close(resChan) defer close(errChan) defer close(reqChan) go func() { - for _, pack := range upgradablePacks { + for _, pack := range updatablePacks { reqChan <- pack } }() @@ -431,7 +431,7 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta timeout := time.After(30 * 60 * time.Second) concurrency := 10 tasks := util.GenWorkers(concurrency) - for range upgradablePacks { + for range updatablePacks { tasks <- func() { select { case pack := <-reqChan: @@ -459,7 +459,7 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta // { DetectedCveID{} : [package] } cvePackages := make(map[DetectedCveID][]string) errs := []error{} - for i := 0; i < len(upgradablePacks); i++ { + for i := 0; i < len(updatablePacks); i++ { select { case response := <-resChan: o.Packages[response.pack.Name] = *response.pack @@ -474,7 +474,7 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta cvePackages[cve] = packNames } o.log.Infof("(%d/%d) Scanned %s: %s", - i+1, len(upgradablePacks), response.pack.Name, cves) + i+1, len(updatablePacks), response.pack.Name, cves) case err := <-errChan: errs = append(errs, err) case <-timeout: @@ -500,7 +500,7 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta } // Update meta package information of changelog cache to the latest one. - meta.Packs = upgradablePacks + meta.Packs = updatablePacks if err := cache.DB.RefreshMeta(*meta); err != nil { return nil, err } @@ -664,7 +664,7 @@ func (o *debian) parseChangelog(changelog, name, ver string, confidence models.C clog := models.Changelog{ Contents: strings.Join(buf, "\n"), - Method: string(confidence.DetectionMethod), + Method: confidence.DetectionMethod, } pack := o.Packages[name] pack.Changelog = clog diff --git a/scan/redhat.go b/scan/redhat.go index 2a8e28d6..c1f12708 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -18,6 +18,7 @@ along with this program. If not, see . package scan import ( + "bufio" "fmt" "regexp" "strings" @@ -27,7 +28,7 @@ import ( "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" - "github.com/k0kubun/pp" + ver "github.com/knqyf263/go-rpm-version" ) // inherit OsTypeInterface @@ -147,14 +148,14 @@ func (o *redhat) checkIfSudoNoPasswd() error { if majorVersion < 6 { cmds = []cmd{ {"yum --color=never repolist", zero}, - {"yum --color=never check-update", []int{0, 100}}, + // {"yum --color=never check-update", []int{0, 100}}, {"yum --color=never list-security --security", zero}, {"yum --color=never info-security", zero}, } } else { cmds = []cmd{ {"yum --color=never repolist", zero}, - {"yum --color=never check-update", []int{0, 100}}, + // {"yum --color=never check-update", []int{0, 100}}, {"yum --color=never --security updateinfo list updates", zero}, {"yum --color=never --security updateinfo updates", zero}, } @@ -174,12 +175,11 @@ func (o *redhat) checkIfSudoNoPasswd() error { return nil } -// CentOS 6, 7 ... yum-plugin-changelog +// CentOS 6, 7 ... yum-plugin-changelog, yum-utils // RHEL 5 ... yum-security // RHEL 6, 7 ... - // Amazon ... - func (o *redhat) checkDependencies() error { - var packName string if o.Distro.Family == config.Amazon { return nil } @@ -207,12 +207,14 @@ func (o *redhat) checkDependencies() error { } } + //TODO Check if yum-plugin-changelog is installed when scan with --changelog option on Amazon,RHEL, Oracle + var packNames []string switch o.Distro.Family { case config.CentOS: - packName = "yum-plugin-changelog" + packNames = []string{"yum-plugin-changelog", "yum-utils"} case config.RedHat, config.Oracle: if majorVersion < 6 { - packName = "yum-security" + packNames = []string{"yum-security"} } else { // yum-plugin-security is installed by default on RHEL6, 7 return nil @@ -221,27 +223,45 @@ func (o *redhat) checkDependencies() error { return fmt.Errorf("Not implemented yet: %s", o.Distro) } - cmd := "rpm -q " + packName - if r := o.exec(cmd, noSudo); !r.isSuccess() { - msg := fmt.Sprintf("%s is not installed", packName) - o.log.Errorf(msg) - return fmt.Errorf(msg) + for _, name := range packNames { + cmd := "rpm -q " + name + if r := o.exec(cmd, noSudo); !r.isSuccess() { + msg := fmt.Sprintf("%s is not installed", name) + o.log.Errorf(msg) + return fmt.Errorf(msg) + } } - o.log.Infof("Dependencies... Pass") + o.log.Infof("Dependencies ... Pass") return nil } func (o *redhat) scanPackages() error { - var err error - var packs []models.Package - if packs, err = o.scanInstalledPackages(); err != nil { + installed, err := o.scanInstalledPackages() + if err != nil { o.log.Errorf("Failed to scan installed packages") return err } - o.setPackages(models.NewPackages(packs...)) + + updatable, err := o.scanUpdatablePackages() + if err != nil { + o.log.Errorf("Failed to scan installed packages") + return err + } + installed.MergeNewVersion(updatable) + o.setPackages(installed) + + if config.Conf.PackageListOnly { + return nil + } + + //TODO Cache changelogs to bolt + //TODO --with-changelog + if err := o.fillChangelogs(updatable); err != nil { + return nil + } var vinfos models.VulnInfos - if vinfos, err = o.scanVulnInfos(); err != nil { + if vinfos, err = o.scanUnsecurePackages(updatable); err != nil { o.log.Errorf("Failed to scan vulnerable packages") return err } @@ -249,406 +269,360 @@ func (o *redhat) scanPackages() error { return nil } -func (o *redhat) scanInstalledPackages() (installed []models.Package, err error) { - cmd := "rpm -qa --queryformat '%{NAME}\t%{EPOCHNUM}\t%{VERSION}\t%{RELEASE}\n'" +func (o *redhat) scanInstalledPackages() (models.Packages, error) { + installed := models.Packages{} + cmd := "rpm -qa --queryformat '%{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE} %{ARCH}\n'" r := o.exec(cmd, noSudo) if r.isSuccess() { - // e.g. - // openssl 1.0.1e 30.el6.11 + // openssl 0 1.0.1e 30.el6.11 x86_64 lines := strings.Split(r.Stdout, "\n") for _, line := range lines { if trimed := strings.TrimSpace(line); len(trimed) != 0 { - var pack models.Package - if pack, err = o.parseScannedPackagesLine(line); err != nil { - return + pack, err := o.parseInstalledPackagesLine(line) + if err != nil { + return nil, err } - installed = append(installed, pack) + installed[pack.Name] = pack } } - return + return installed, nil } - return nil, fmt.Errorf( - "Scan packages failed. status: %d, stdout: %s, stderr: %s", + return nil, fmt.Errorf("Scan packages failed. status: %d, stdout: %s, stderr: %s", r.ExitStatus, r.Stdout, r.Stderr) + } -func (o *redhat) parseScannedPackagesLine(line string) (models.Package, error) { +func (o *redhat) parseInstalledPackagesLine(line string) (models.Package, error) { fields := strings.Fields(line) - if len(fields) != 4 { + if len(fields) != 5 { return models.Package{}, fmt.Errorf("Failed to parse package line: %s", line) } ver := "" - if fields[1] == "0" { + epoch := fields[1] + if epoch == "0" { ver = fields[2] } else { - ver = fmt.Sprintf("%s:%s", fields[1], fields[2]) + ver = fmt.Sprintf("%s:%s", epoch, fields[2]) } + return models.Package{ Name: fields[0], Version: ver, Release: fields[3], + Arch: fields[4], }, nil } -func (o *redhat) scanVulnInfos() (models.VulnInfos, error) { - if o.Distro.Family != config.CentOS { - // Amazon, RHEL, Oracle Linux has yum updateinfo as default - // yum updateinfo can collenct vendor advisory information. - return o.scanUnsecurePackagesUsingYumPluginSecurity() - } - // CentOS does not have security channel... - // So, yum check-update then parse chnagelog. - return o.scanUnsecurePackagesUsingYumCheckUpdate() -} - -// For CentOS -func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, error) { - cmd := "LANGUAGE=en_US.UTF-8 yum --color=never %s check-update" - if o.getServerInfo().Enablerepo != "" { - cmd = fmt.Sprintf(cmd, "--enablerepo="+o.getServerInfo().Enablerepo) - } else { - cmd = fmt.Sprintf(cmd, "") +func (o *redhat) scanUpdatablePackages() (models.Packages, error) { + cmd := "repoquery --all --pkgnarrow=updates --qf='%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{REPO}'" + for _, repo := range o.getServerInfo().Enablerepo { + cmd += " --enablerepo=" + repo } - r := o.exec(util.PrependProxyEnv(cmd), noSudo) - if !r.isSuccess(0, 100) { - //returns an exit code of 100 if there are available updates. + r := o.exec(util.PrependProxyEnv(cmd), o.sudo()) + if !r.isSuccess() { return nil, fmt.Errorf("Failed to SSH: %s", r) } - // get Updateble package name, installed, candidate version. - packages, err := o.parseYumCheckUpdateLines(r.Stdout) - if err != nil { - return nil, fmt.Errorf("Failed to parse %s. err: %s", cmd, err) - } - o.log.Debugf("%s", pp.Sprintf("%v", packages)) - - // set candidate version info - o.Packages.MergeNewVersion(packages) - - // Collect CVE-IDs in changelog - type PackageCveIDs struct { - Package models.Package - CveIDs []string - } - - allChangelog, err := o.getAllChangelog(packages) - if err != nil { - o.log.Errorf("Failed to getAllchangelog. err: %s", err) - return nil, err - } - - // { packageName: changelog-lines } - var rpm2changelog map[string]*string - rpm2changelog, err = o.divideChangelogByPackage(allChangelog) - if err != nil { - return nil, fmt.Errorf("Failed to parseAllChangelog. err: %s", err) - } - - for name, clog := range rpm2changelog { - for _, p := range o.Packages { - n := fmt.Sprintf("%s-%s-%s", p.Name, p.NewVersion, p.NewRelease) - if name == n { - p.Changelog = models.Changelog{ - Contents: *clog, - Method: models.ChangelogExactMatchStr, - } - o.Packages[p.Name] = p - break - } - } - } - - var results []PackageCveIDs - i := 0 - for name := range packages { - changelog := o.getChangelogCVELines(rpm2changelog, packages[name]) - - // Collect unique set of CVE-ID in each changelog - uniqueCveIDMap := make(map[string]bool) - lines := strings.Split(changelog, "\n") - for _, line := range lines { - cveIDs := o.parseYumUpdateinfoLineToGetCveIDs(line) - for _, c := range cveIDs { - uniqueCveIDMap[c] = true - } - } - - // keys - var cveIDs []string - for k := range uniqueCveIDMap { - cveIDs = append(cveIDs, k) - } - p := PackageCveIDs{ - Package: packages[name], - CveIDs: cveIDs, - } - results = append(results, p) - - o.log.Infof("(%d/%d) Scanned %s-%s-%s -> %s-%s : %s", - i+1, - len(packages), - p.Package.Name, - p.Package.Version, - p.Package.Release, - p.Package.NewVersion, - p.Package.NewRelease, - p.CveIDs) - i++ - } - - // transform datastructure - // - From - // [ - // { - // Pack: models.Packages, - // CveIDs: []string, - // }, - // ] - // - To - // map { - // CveID: models.Packages{} - // } - cveIDPackages := make(map[string]models.Packages) - for _, res := range results { - for _, cveID := range res.CveIDs { - if packages, ok := cveIDPackages[cveID]; ok { - packages[res.Package.Name] = res.Package - cveIDPackages[cveID] = packages - } else { - cveIDPackages[cveID] = models.NewPackages(res.Package) - } - } - } - - vinfos := models.VulnInfos{} - for cveID, packs := range cveIDPackages { - names := []string{} - for name := range packs { - names = append(names, name) - } - - // Amazon, RHEL do not use this method, so VendorAdvisory do not set. - vinfos[cveID] = models.VulnInfo{ - CveID: cveID, - PackageNames: names, - Confidence: models.ChangelogExactMatch, - } - } - return vinfos, nil + // Collect Updateble packages, installed, candidate version and repository. + return o.parseUpdatablePacksLines(r.Stdout) } -// parseYumCheckUpdateLines parse yum check-update to get package name, candidate version -func (o *redhat) parseYumCheckUpdateLines(stdout string) (models.Packages, error) { - results := models.Packages{} - needToParse := false +// parseUpdatablePacksLines parse the stdout of repoquery to get package name, candidate version +func (o *redhat) parseUpdatablePacksLines(stdout string) (models.Packages, error) { + updatable := models.Packages{} lines := strings.Split(stdout, "\n") for _, line := range lines { - // update information of packages begin after blank line. - if trimed := strings.TrimSpace(line); len(trimed) == 0 { - needToParse = true + // TODO remove + // if strings.HasPrefix(line, "Obsoleting") || + // strings.HasPrefix(line, "Security:") { + // // see https://github.com/future-architect/vuls/issues/165 + // continue + // } + if len(strings.TrimSpace(line)) == 0 { continue } - if needToParse { - if strings.HasPrefix(line, "Obsoleting") || - strings.HasPrefix(line, "Security:") { - // see https://github.com/future-architect/vuls/issues/165 - continue - } - candidate, err := o.parseYumCheckUpdateLine(line) - if err != nil { - return results, err - } - - installed, found := o.Packages[candidate.Name] - if !found { - o.log.Warnf("Not found the package in rpm -qa. candidate: %s-%s-%s", - candidate.Name, candidate.Version, candidate.Release) - results[candidate.Name] = candidate - continue - } - installed.NewVersion = candidate.NewVersion - installed.NewRelease = candidate.NewRelease - installed.Repository = candidate.Repository - results[installed.Name] = installed + pack, err := o.parseUpdatablePacksLine(line) + if err != nil { + return updatable, err } + updatable[pack.Name] = pack } - return results, nil + return updatable, nil } -func (o *redhat) parseYumCheckUpdateLine(line string) (models.Package, error) { +func (o *redhat) parseUpdatablePacksLine(line string) (models.Package, error) { fields := strings.Fields(line) - if len(fields) < 3 { - return models.Package{}, fmt.Errorf("Unknown format: %s", line) + if len(fields) < 5 { + return models.Package{}, fmt.Errorf("Unknown format: %s, fields: %s", line, fields) } - splitted := strings.Split(fields[0], ".") - packName := "" - if len(splitted) == 1 { - packName = fields[0] + + ver := "" + epoch := fields[1] + if epoch == "0" { + ver = fields[2] } else { - packName = strings.Join(strings.Split(fields[0], ".")[0:(len(splitted)-1)], ".") + ver = fmt.Sprintf("%s:%s", epoch, fields[2]) } - verfields := strings.Split(fields[1], "-") - if len(verfields) != 2 { - return models.Package{}, fmt.Errorf("Unknown format: %s", line) - } - release := verfields[1] - repos := strings.Join(fields[2:len(fields)], " ") + repos := strings.Join(fields[4:len(fields)], " ") - return models.Package{ - Name: packName, - NewVersion: verfields[0], - NewRelease: release, + p := models.Package{ + Name: fields[0], + NewVersion: ver, + NewRelease: fields[3], Repository: repos, - }, nil -} - -func (o *redhat) mkPstring() *string { - str := "" - return &str -} - -func (o *redhat) regexpReplace(src string, pat string, rep string) string { - re := regexp.MustCompile(pat) - return re.ReplaceAllString(src, rep) -} - -var changeLogCVEPattern = regexp.MustCompile(`CVE-[0-9]+-[0-9]+`) - -func (o *redhat) getChangelogCVELines(rpm2changelog map[string]*string, pack models.Package) string { - rpm := fmt.Sprintf("%s-%s-%s", pack.Name, pack.NewVersion, pack.NewRelease) - retLine := "" - if rpm2changelog[rpm] != nil { - lines := strings.Split(*rpm2changelog[rpm], "\n") - for _, line := range lines { - if changeLogCVEPattern.MatchString(line) { - retLine += fmt.Sprintf("%s\n", line) - } - } } - return retLine + return p, nil } -func (o *redhat) divideChangelogByPackage(allChangelog string) (map[string]*string, error) { - var majorVersion int - var err error - if o.Distro.Family == config.CentOS { - majorVersion, err = o.Distro.MajorVersion() - if err != nil { - return nil, fmt.Errorf("Not implemented yet: %s, err: %s", o.Distro, err) +func (o *redhat) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) { + if o.Distro.Family != config.CentOS { + // Amazon, RHEL, Oracle Linux has yum updateinfo as default + // yum updateinfo can collenct vendor advisory information. + return o.scanCveIDsByCommands(updatable) + } + + // Parse chnagelog because CentOS does not have security channel... + return o.scanCveIDsInChangelog(updatable) +} + +func (o *redhat) fillChangelogs(updatables models.Packages) error { + names := []string{} + for name := range updatables { + names = append(names, name) + } + + if err := o.fillDiffChangelogs(names); err != nil { + return err + } + + emptyChangelogPackNames := []string{} + for _, pack := range o.Packages { + if pack.NewVersion != "" && pack.Changelog.Contents == "" { + emptyChangelogPackNames = append(emptyChangelogPackNames, pack.Name) } } - orglines := strings.Split(allChangelog, "\n") - tmpline := "" - var lines []string - var prev, now bool - for i := range orglines { - if majorVersion == 5 { - /* for CentOS5 (yum-util < 1.1.20) */ - prev = false - now = false - if 0 < i { - prev, err = o.isRpmPackageNameLine(orglines[i-1]) - if err != nil { - return nil, err - } - } - now, err = o.isRpmPackageNameLine(orglines[i]) - if err != nil { - return nil, err - } - if prev && now { - tmpline = fmt.Sprintf("%s, %s", tmpline, orglines[i]) - continue - } - if !prev && now { - tmpline = fmt.Sprintf("%s%s", tmpline, orglines[i]) - continue - } - if tmpline != "" { - lines = append(lines, fmt.Sprintf("%s", tmpline)) - tmpline = "" - } - lines = append(lines, fmt.Sprintf("%s", orglines[i])) - } else { - /* for CentOS6,7 (yum-util >= 1.1.20) */ - line := orglines[i] - line = o.regexpReplace(line, `^ChangeLog for: `, "") - line = o.regexpReplace(line, `^\*\*\sNo\sChangeLog\sfor:.*`, "") - lines = append(lines, line) + i := 0 + for _, name := range emptyChangelogPackNames { + i++ + o.log.Infof("(%d/%d) Fetched Changelogs %s", i, len(emptyChangelogPackNames), name) + if err := o.fillDiffChangelogs([]string{name}); err != nil { + return err } } - rpm2changelog := make(map[string]*string) - writePointer := o.mkPstring() - for _, line := range lines { - match, err := o.isRpmPackageNameLine(line) - if err != nil { - return nil, err - } - if match { - rpms := strings.Split(line, ",") - pNewString := o.mkPstring() - writePointer = pNewString - for _, rpm := range rpms { - rpm = strings.TrimSpace(rpm) - rpm = o.regexpReplace(rpm, `\.(i386|i486|i586|i686|k6|athlon|x86_64|noarch|ppc|alpha|sparc)$`, "") - if ss := strings.Split(rpm, ":"); 1 < len(ss) { - epoch := ss[0] - packVersion := strings.Join(ss[1:len(ss)], ":") - if sss := strings.Split(packVersion, "-"); 2 < len(sss) { - version := strings.Join(sss[len(sss)-2:len(sss)], "-") - name := strings.Join(sss[0:len(sss)-2], "-") - rpm = fmt.Sprintf("%s-%s:%s", name, epoch, version) - } - } - - rpm2changelog[rpm] = pNewString - } - } else { - if strings.HasPrefix(line, "Dependencies Resolved") { - return rpm2changelog, nil - } - *writePointer += fmt.Sprintf("%s\n", line) - } - } - return rpm2changelog, nil + return nil } -// CentOS -func (o *redhat) getAllChangelog(packages models.Packages) (stdout string, err error) { - packageNames := "" - for _, pack := range packages { - packageNames += fmt.Sprintf("%s ", pack.Name) - } - - command := "" - if 0 < len(config.Conf.HTTPProxy) { - command += util.ProxyEnv() - } - +func (o *redhat) getAvailableChangelogs(packNames []string) (map[string]string, error) { yumopts := "" - if o.getServerInfo().Enablerepo != "" { - yumopts = " --enablerepo=" + o.getServerInfo().Enablerepo + if 0 < len(o.getServerInfo().Enablerepo) { + yumopts = " --enablerepo=" + strings.Join(o.getServerInfo().Enablerepo, ",") } if config.Conf.SkipBroken { yumopts += " --skip-broken" } + cmd := `yum --color=never %s changelog all %s | grep -A 10000 '==================== Available Packages ===================='` + cmd = fmt.Sprintf(cmd, yumopts, strings.Join(packNames, " ")) - // yum update --changelog doesn't have --color option. - command += fmt.Sprintf(" LANGUAGE=en_US.UTF-8 yum --changelog --assumeno update %s ", yumopts) + packageNames - - r := o.exec(command, sudo) + r := o.exec(util.PrependProxyEnv(cmd), noSudo) if !r.isSuccess(0, 1) { - return "", fmt.Errorf( - "Failed to get changelog. status: %d, stdout: %s, stderr: %s", - r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } - return strings.Replace(r.Stdout, "\r", "", -1), nil + + return o.divideChangelogsIntoEachPackages(r.Stdout), nil +} + +// Divide available change logs of all updatable packages into each package's changelog +func (o *redhat) divideChangelogsIntoEachPackages(stdout string) map[string]string { + changelogs := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(stdout)) + + crlf, newBlock := false, true + packNameVer, contents := "", []string{} + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "==================== Available Packages ====================") { + continue + } + if newBlock { + left := strings.Fields(line)[0] + // ss := strings.Split(left, ".") + // packNameVer = strings.Join(ss[0:len(ss)-1], ".") + packNameVer = left + newBlock = false + continue + } + if len(strings.TrimSpace(line)) == 0 { + if crlf { + changelogs[packNameVer] = strings.Join(contents, "\n") + packNameVer = "" + contents = []string{} + newBlock = true + crlf = false + } else { + contents = append(contents, line) + crlf = true + } + } else { + contents = append(contents, line) + crlf = false + } + } + if 0 < len(contents) { + changelogs[packNameVer] = strings.Join(contents, "\n") + } + return changelogs +} + +func (o *redhat) fillDiffChangelogs(packNames []string) error { + changelogs, err := o.getAvailableChangelogs(packNames) + if err != nil { + return err + } + + for s := range changelogs { + // name, pack, found := o.Packages.FindOne(func(p models.Package) bool { + name, pack, found := o.Packages.FindOne(func(p models.Package) bool { + var epochNameVerRel string + if index := strings.Index(p.NewVersion, ":"); 0 < index { + epoch := p.NewVersion[0:index] + ver := p.NewVersion[index+1 : len(p.NewVersion)] + epochNameVerRel = fmt.Sprintf("%s:%s-%s", + epoch, p.Name, ver) + } else { + epochNameVerRel = fmt.Sprintf("%s-%s", + p.Name, p.NewVersion) + } + return strings.HasPrefix(s, epochNameVerRel) + }) + + if found { + diff, err := o.getDiffChangelog(pack, changelogs[s]) + detectionMethod := models.ChangelogExactMatchStr + + if err != nil { + o.log.Debug(err) + // Try without epoch + if index := strings.Index(pack.Version, ":"); 0 < index { + pack.Version = pack.Version[index+1 : len(pack.Version)] + o.log.Debug("Try without epoch", pack) + diff, err = o.getDiffChangelog(pack, changelogs[s]) + if err != nil { + o.log.Debugf("Failed to find the version in changelog: %s-%s-%s", + pack.Name, pack.Version, pack.Release) + detectionMethod = models.FailedToFindVersionInChangelog + } else { + o.log.Debugf("Found the version in changelog without epoch: %s-%s-%s", + pack.Name, pack.Version, pack.Release) + detectionMethod = models.ChangelogLenientMatchStr + } + } + } + + pack = o.Packages[name] + pack.Changelog = models.Changelog{ + Contents: diff, + Method: models.DetectionMethod(detectionMethod), + } + o.Packages[name] = pack + } + } + return nil +} + +func (o *redhat) getDiffChangelog(pack models.Package, availableChangelog string) (string, error) { + installedVer := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) + scanner := bufio.NewScanner(strings.NewReader(availableChangelog)) + diff := []string{} + found := false + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "* ") { + diff = append(diff, line) + continue + } + + // openssh on RHEL + // openssh-server-6.6.1p1-35.el7_3.x86_64 rhui-rhel-7-server-rhui-rpms + // Wed Mar 1 21:00:00 2017 Jakub Jelen - 6.6.1p1-35 + 0.9.3-9 + ss := strings.Split(line, " + ") + if 1 < len(ss) { + line = ss[0] + } + + ss = strings.Split(line, " ") + if len(ss) < 2 { + diff = append(diff, line) + continue + } + v := ss[len(ss)-1] + v = strings.TrimPrefix(v, "-") + v = strings.TrimPrefix(v, "[") + v = strings.TrimSuffix(v, "]") + version := ver.NewVersion(v) + if installedVer.Equal(version) || installedVer.GreaterThan(version) { + found = true + break + } + diff = append(diff, line) + } + + if len(diff) == 0 || !found { + return availableChangelog, + fmt.Errorf("Failed to find the version in changelog: %s-%s-%s", + pack.Name, pack.Version, pack.Release) + } + return strings.TrimSpace(strings.Join(diff, "\n")), nil +} + +func (o *redhat) scanCveIDsInChangelog(updatable models.Packages) (models.VulnInfos, error) { + packCveIDs := make(map[string][]string) + for name := range updatable { + cveIDs := []string{} + pack := o.Packages[name] + if pack.Changelog.Method == models.FailedToFindVersionInChangelog { + continue + } + scanner := bufio.NewScanner(strings.NewReader(pack.Changelog.Contents)) + for scanner.Scan() { + if matches := cveRe.FindAllString(scanner.Text(), -1); 0 < len(matches) { + for _, m := range matches { + cveIDs = util.AppendIfMissing(cveIDs, m) + } + } + } + packCveIDs[name] = cveIDs + } + + // transform datastructure + // - From + // "packname": []{"CVE-2017-1111", ".../ + // + // - To + // map { + // "CVE-2017-1111": "packname", + // } + vinfos := models.VulnInfos{} + for name, cveIDs := range packCveIDs { + for _, cid := range cveIDs { + if v, ok := vinfos[cid]; ok { + v.PackageNames = append(v.PackageNames, name) + vinfos[cid] = v + } else { + vinfos[cid] = models.VulnInfo{ + CveID: cid, + PackageNames: []string{name}, + Confidence: models.ChangelogExactMatch, + } + } + } + } + return vinfos, nil } type distroAdvisoryCveIDs struct { @@ -658,7 +632,7 @@ type distroAdvisoryCveIDs struct { // Scaning unsecure packages using yum-plugin-security. // Amazon, RHEL, Oracle Linux -func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, error) { +func (o *redhat) scanCveIDsByCommands(updatable models.Packages) (models.VulnInfos, error) { if o.Distro.Family == config.CentOS { // CentOS has no security channel. // So use yum check-update && parse changelog @@ -689,22 +663,6 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, } advIDPackNamesList, err := o.parseYumUpdateinfoListAvailable(r.Stdout) - // get package name, version, rel to be upgrade. - cmd = "LANGUAGE=en_US.UTF-8 yum --color=never check-update" - r = o.exec(util.PrependProxyEnv(cmd), o.sudo()) - if !r.isSuccess(0, 100) { - //returns an exit code of 100 if there are available updates. - return nil, fmt.Errorf("Failed to SSH: %s", r) - } - updatable, err := o.parseYumCheckUpdateLines(r.Stdout) - if err != nil { - return nil, fmt.Errorf("Failed to parse %s. err: %s", cmd, err) - } - o.log.Debugf("%s", pp.Sprintf("%v", updatable)) - - // set candidate version info - o.Packages.MergeNewVersion(updatable) - dict := make(map[string]models.Packages) for _, advIDPackNames := range advIDPackNamesList { packages := models.Packages{} @@ -796,8 +754,6 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID for cveID := range cveIDsSetInThisSection { foundCveIDs = append(foundCveIDs, cveID) } - //TODO remove - // sort.Strings(foundCveIDs) result = append(result, distroAdvisoryCveIDs{ DistroAdvisory: advisory, @@ -883,24 +839,6 @@ func (o *redhat) changeSectionState(state int) (newState int) { return newState } -var rpmPackageArchPattern = regexp.MustCompile( - `^[^ ]+\.(i386|i486|i586|i686|k6|athlon|x86_64|noarch|ppc|alpha|sparc)$`) - -func (o *redhat) isRpmPackageNameLine(line string) (bool, error) { - s := strings.TrimPrefix(line, "ChangeLog for: ") - ss := strings.Split(s, ", ") - if len(ss) == 0 { - return false, nil - } - for _, s := range ss { - s = strings.TrimRight(s, " \r\n") - if !rpmPackageArchPattern.MatchString(s) { - return false, nil - } - } - return true, nil -} - var yumCveIDPattern = regexp.MustCompile(`(CVE-\d{4}-\d{4,})`) func (o *redhat) parseYumUpdateinfoLineToGetCveIDs(line string) []string { @@ -1032,9 +970,10 @@ func (o *redhat) clone() osTypeInterface { func (o *redhat) sudo() bool { switch o.Distro.Family { - case config.Amazon: + case config.Amazon, config.CentOS: return false default: + // RHEL return true } } diff --git a/scan/redhat_test.go b/scan/redhat_test.go index f3bf219a..e5414fe7 100644 --- a/scan/redhat_test.go +++ b/scan/redhat_test.go @@ -6,9 +6,7 @@ it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License @@ -20,6 +18,7 @@ package scan import ( "reflect" "sort" + "strings" "testing" "time" @@ -42,7 +41,7 @@ func TestParseScanedPackagesLineRedhat(t *testing.T) { pack models.Package }{ { - "openssl 0 1.0.1e 30.el6.11", + "openssl 0 1.0.1e 30.el6.11 x86_64", models.Package{ Name: "openssl", Version: "1.0.1e", @@ -50,7 +49,7 @@ func TestParseScanedPackagesLineRedhat(t *testing.T) { }, }, { - "Percona-Server-shared-56 1 5.6.19 rel67.0.el6", + "Percona-Server-shared-56 1 5.6.19 rel67.0.el6 x84_64", models.Package{ Name: "Percona-Server-shared-56", Version: "1:5.6.19", @@ -60,7 +59,7 @@ func TestParseScanedPackagesLineRedhat(t *testing.T) { } for _, tt := range packagetests { - p, _ := r.parseScannedPackagesLine(tt.in) + p, _ := r.parseInstalledPackagesLine(tt.in) if p.Name != tt.pack.Name { t.Errorf("name: expected %s, actual %s", tt.pack.Name, p.Name) } @@ -262,54 +261,6 @@ func TestIsDescriptionLine(t *testing.T) { } } -func TestIsRpmPackageNameLine(t *testing.T) { - r := newRedhat(config.ServerInfo{}) - var tests = []struct { - in string - found bool - }{ - { - "stunnel-4.15-2.el5.2.i386", - true, - }, - { - "iproute-2.6.18-15.el5.i386", - true, - }, - { - "1:yum-updatesd-0.9-6.el5_10.noarch", - true, - }, - { - "glibc-2.12-1.192.el6.x86_64", - true, - }, - { - " glibc-2.12-1.192.el6.x86_64", - false, - }, - { - "glibc-2.12-1.192.el6.x86_64, iproute-2.6.18-15.el5.i386", - true, - }, - { - "k6 hoge.i386", - false, - }, - { - "triathlon", - false, - }, - } - - for i, tt := range tests { - found, err := r.isRpmPackageNameLine(tt.in) - if tt.found != found { - t.Errorf("[%d] line: %s, expected %t, actual %t, err %v", i, tt.in, tt.found, found, err) - } - } -} - func TestParseYumUpdateinfoToGetSeverity(t *testing.T) { r := newRedhat(config.ServerInfo{}) var tests = []struct { @@ -672,57 +623,64 @@ Description : Package updates are available for Amazon Linux AMI that fix the } } +func TestParseYumCheckUpdateLine(t *testing.T) { + r := newRedhat(config.ServerInfo{}) + r.Distro = config.Distro{Family: "centos"} + var tests = []struct { + in string + out models.Package + }{ + { + "zlib 0 1.2.7 17.el7 rhui-REGION-rhel-server-releases", + models.Package{ + Name: "zlib", + NewVersion: "1.2.7", + NewRelease: "17.el7", + Repository: "rhui-REGION-rhel-server-releases", + }, + }, + { + "shadow-utils 2 4.1.5.1 24.el7 rhui-REGION-rhel-server-releases", + models.Package{ + Name: "shadow-utils", + NewVersion: "2:4.1.5.1", + NewRelease: "24.el7", + Repository: "rhui-REGION-rhel-server-releases", + }, + }, + } + + for _, tt := range tests { + aPack, err := r.parseUpdatablePacksLine(tt.in) + if err != nil { + t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in) + return + } + if !reflect.DeepEqual(tt.out, aPack) { + e := pp.Sprintf("%v", tt.out) + a := pp.Sprintf("%v", aPack) + t.Errorf("expected %s, actual %s", e, a) + } + } +} + func TestParseYumCheckUpdateLines(t *testing.T) { r := newRedhat(config.ServerInfo{}) r.Distro = config.Distro{Family: "centos"} - stdout := `Loaded plugins: changelog, fastestmirror, keys, protect-packages, protectbase, security -Loading mirror speeds from cached hostfile - * base: mirror.fairway.ne.jp - * epel: epel.mirror.srv.co.ge - * extras: mirror.fairway.ne.jp - * updates: mirror.fairway.ne.jp -0 packages excluded due to repository protections - -audit-libs.x86_64 2.3.7-5.el6 base -bash.x86_64 4.1.2-33.el6_7.1 updates -Obsoleting Packages -python-libs.i686 2.6.6-64.el6 rhui-REGION-rhel-server-releases - python-ordereddict.noarch 1.1-3.el6ev installed -bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates -pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 -` + stdout := `audit-libs 0 2.3.7 5.el6 base +bash 0 4.1.2 33.el6_7.1 updates +python-libs 0 2.6.6 64.el6 rhui-REGION-rhel-server-releases +python-ordereddict 0 1.1 3.el6ev installed +bind-utils 30 9.3.6 25.P1.el5_11.8 updates +pytalloc 0 2.0.7 2.el6 @CentOS 6.5/6.5` r.setPackages(models.NewPackages( - models.Package{ - Name: "audit-libs", - Version: "2.3.6", - Release: "4.el6", - }, - models.Package{ - Name: "bash", - Version: "4.1.1", - Release: "33", - }, - models.Package{ - Name: "python-libs", - Version: "2.6.0", - Release: "1.1-0", - }, - models.Package{ - Name: "python-ordereddict", - Version: "1.0", - Release: "1", - }, - models.Package{ - Name: "bind-utils", - Version: "1.0", - Release: "1", - }, - models.Package{ - Name: "pytalloc", - Version: "2.0.1", - Release: "0", - }, + models.Package{Name: "audit-libs"}, + models.Package{Name: "bash"}, + models.Package{Name: "python-libs"}, + models.Package{Name: "python-ordereddict"}, + models.Package{Name: "bind-utils"}, + models.Package{Name: "pytalloc"}, )) var tests = []struct { in string @@ -733,48 +691,36 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 models.NewPackages( models.Package{ Name: "audit-libs", - Version: "2.3.6", - Release: "4.el6", NewVersion: "2.3.7", NewRelease: "5.el6", Repository: "base", }, models.Package{ Name: "bash", - Version: "4.1.1", - Release: "33", NewVersion: "4.1.2", NewRelease: "33.el6_7.1", Repository: "updates", }, models.Package{ Name: "python-libs", - Version: "2.6.0", - Release: "1.1-0", NewVersion: "2.6.6", NewRelease: "64.el6", Repository: "rhui-REGION-rhel-server-releases", }, models.Package{ Name: "python-ordereddict", - Version: "1.0", - Release: "1", NewVersion: "1.1", NewRelease: "3.el6ev", Repository: "installed", }, models.Package{ Name: "bind-utils", - Version: "1.0", - Release: "1", NewVersion: "30:9.3.6", NewRelease: "25.P1.el5_11.8", Repository: "updates", }, models.Package{ Name: "pytalloc", - Version: "2.0.1", - Release: "0", NewVersion: "2.0.7", NewRelease: "2.el6", Repository: "@CentOS 6.5/6.5", @@ -784,7 +730,7 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 } for _, tt := range tests { - packages, err := r.parseYumCheckUpdateLines(tt.in) + packages, err := r.parseUpdatablePacksLines(tt.in) if err != nil { t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in) return @@ -802,29 +748,13 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 func TestParseYumCheckUpdateLinesAmazon(t *testing.T) { r := newRedhat(config.ServerInfo{}) r.Distro = config.Distro{Family: "amazon"} - stdout := `Loaded plugins: priorities, update-motd, upgrade-helper -34 package(s) needed for security, out of 71 available - -bind-libs.x86_64 32:9.8.2-0.37.rc1.45.amzn1 amzn-main -java-1.7.0-openjdk.x86_64 1.7.0.95-2.6.4.0.65.amzn1 amzn-main -if-not-architecture 100-200 amzn-main -` + stdout := `bind-libs 32 9.8.2 0.37.rc1.45.amzn1 amzn-main +java-1.7.0-openjdk 0 1.7.0.95 2.6.4.0.65.amzn1 amzn-main +if-not-architecture 0 100 200 amzn-main` r.Packages = models.NewPackages( - models.Package{ - Name: "bind-libs", - Version: "9.8.0", - Release: "0.33.rc1.45.amzn1", - }, - models.Package{ - Name: "java-1.7.0-openjdk", - Version: "1.7.0.0", - Release: "2.6.4.0.0.amzn1", - }, - models.Package{ - Name: "if-not-architecture", - Version: "10", - Release: "20", - }, + models.Package{Name: "bind-libs"}, + models.Package{Name: "java-1.7.0-openjdk"}, + models.Package{Name: "if-not-architecture"}, ) var tests = []struct { in string @@ -835,24 +765,18 @@ if-not-architecture 100-200 amzn-main models.NewPackages( models.Package{ Name: "bind-libs", - Version: "9.8.0", - Release: "0.33.rc1.45.amzn1", NewVersion: "32:9.8.2", NewRelease: "0.37.rc1.45.amzn1", Repository: "amzn-main", }, models.Package{ Name: "java-1.7.0-openjdk", - Version: "1.7.0.0", - Release: "2.6.4.0.0.amzn1", NewVersion: "1.7.0.95", NewRelease: "2.6.4.0.65.amzn1", Repository: "amzn-main", }, models.Package{ Name: "if-not-architecture", - Version: "10", - Release: "20", NewVersion: "100", NewRelease: "200", Repository: "amzn-main", @@ -862,7 +786,7 @@ if-not-architecture 100-200 amzn-main } for _, tt := range tests { - packages, err := r.parseYumCheckUpdateLines(tt.in) + packages, err := r.parseUpdatablePacksLines(tt.in) if err != nil { t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in) return @@ -976,308 +900,407 @@ func TestExtractPackNameVerRel(t *testing.T) { } -const ( - /* for CentOS6,7 (yum-util >= 1.1.20) */ - stdoutCentos6 = `---> Package libaio.x86_64 0:0.3.107-10.el6 will be installed ---> Finished Dependency Resolution - -Changes in packages about to be updated: - -ChangeLog for: binutils-2.20.51.0.2-5.44.el6.x86_64 -* Mon Dec 7 21:00:00 2015 Nick Clifton - 2.20.51.0.2-5.44 -- Backport upstream RELRO fixes. (#1227839) - -** No ChangeLog for: chkconfig-1.3.49.5-1.el6.x86_64 - -ChangeLog for: coreutils-8.4-43.el6.x86_64, coreutils-libs-8.4-43.el6.x86_64 -* Wed Feb 10 21:00:00 2016 Ondrej Vasik - 8.4-43 -- sed should actually be /bin/sed (related #1222140) - -* Wed Jan 6 21:00:00 2016 Ondrej Vasik - 8.4-41 -- colorls.sh,colorls.csh - call utilities with complete path (#1222140) -- mkdir, mkfifo, mknod - respect default umask/acls when - COREUTILS_CHILD_DEFAULT_ACLS envvar is set (to match rhel 7 behaviour, - -ChangeLog for: centos-release-6-8.el6.centos.12.3.x86_64 -* Wed May 18 21:00:00 2016 Johnny Hughes 6-8.el6.centos.12.3 -- CentOS-6.8 Released -- TESTSTRING CVE-0000-0000 - -ChangeLog for: 12:dhclient-4.1.1-51.P1.el6.centos.x86_64, 12:dhcp-common-4.1.1-51.P1.el6.centos.x86_64 -* Tue May 10 21:00:00 2016 Johnny Hughes - 12:4.1.1-51.P1 -- created patch 1000 for CentOS Branding -- replaced vvendor variable with CentOS in the SPEC file -- TESTSTRING CVE-1111-1111 - -* Mon Jan 11 21:00:00 2016 Jiri Popelka - 12:4.1.1-51.P1 -- send unicast request/release via correct interface (#1297445) - -* Thu Dec 3 21:00:00 2015 Jiri Popelka - 12:4.1.1-50.P1 -- Lease table overflow crash. (#1133917) -- Add ignore-client-uids option. (#1196768) -- dhclient-script: it's OK if the arping reply comes from our system. (#1204095) -- VLAN ID is only bottom 12-bits of TCI. (#1259552) -- dhclient: Make sure link-local address is ready in stateless mode. (#1263466) -- dhclient-script: make_resolv_conf(): Keep old nameservers - if server sends domain-name/search, but no nameservers. (#1269595) - -ChangeLog for: file-5.04-30.el6.x86_64, file-libs-5.04-30.el6.x86_64 -* Tue Feb 16 21:00:00 2016 Jan Kaluza 5.04-30 -- fix CVE-2014-3538 (unrestricted regular expression matching) - -* Tue Jan 5 21:00:00 2016 Jan Kaluza 5.04-29 -- fix #1284826 - try to read ELF header to detect corrupted one - -* Wed Dec 16 21:00:00 2015 Jan Kaluza 5.04-28 -- fix #1263987 - fix bugs found by coverity in the patch - -* Thu Nov 26 21:00:00 2015 Jan Kaluza 5.04-27 -- fix CVE-2014-3587 (incomplete fix for CVE-2012-1571) -- fix CVE-2014-3710 (out-of-bounds read in elf note headers) -- fix CVE-2014-8116 (multiple DoS issues (resource consumption)) -- fix CVE-2014-8117 (denial of service issue (resource consumption)) -- fix CVE-2014-9620 (limit the number of ELF notes processed) -- fix CVE-2014-9653 (malformed elf file causes access to uninitialized memory) - - -Dependencies Resolved - -` - /* for CentOS5 (yum-util < 1.1.20) */ - stdoutCentos5 = `---> Package portmap.i386 0:4.0-65.2.2.1 set to be updated ---> Finished Dependency Resolution - -Changes in packages about to be updated: - -libuser-0.54.7-3.el5.i386 -nss_db-2.2-38.el5_11.i386 -* Thu Nov 20 23:00:00 2014 Nalin Dahyabhai - 2.2-38 -- build without strict aliasing (internal build tooling) - -* Sat Nov 15 23:00:00 2014 Nalin Dahyabhai - 2.2-37 -- pull in fix for a memory leak in nss_db (#1163493) - -acpid-1.0.4-12.el5.i386 -* Thu Oct 6 00:00:00 2011 Jiri Skala - 1.0.4-12 -- Resolves: #729769 - acpid dumping useless info to log - -nash-5.1.19.6-82.el5.i386, mkinitrd-5.1.19.6-82.el5.i386 -* Tue Apr 15 00:00:00 2014 Brian C. Lane 5.1.19.6-82 -- Use ! instead of / when searching sysfs for ccis device - Resolves: rhbz#988020 -- Always include ahci module (except on s390) (bcl) - Resolves: rhbz#978245 -- Prompt to recreate default initrd (karsten) - Resolves: rhbz#472764 - -util-linux-2.13-0.59.el5_8.i386 -* Wed Oct 17 00:00:00 2012 Karel Zak 2.13-0.59.el5_8 -- fix #865791 - fdisk fails to partition disk not in use - -* Wed Dec 21 23:00:00 2011 Karel Zak 2.13-0.59 -- fix #768382 - CVE-2011-1675 CVE-2011-1677 util-linux various flaws - -* Wed Oct 26 00:00:00 2011 Karel Zak 2.13-0.58 -- fix #677452 - util-linux fails to build with gettext-0.17 - -30:bind-utils-9.3.6-25.P1.el5_11.8.i386, 30:bind-libs-9.3.6-25.P1.el5_11.8.i386 -* Mon Mar 14 23:00:00 2016 Tomas Hozza - 30:9.3.6-25.P1.8 -- Fix issue with patch for CVE-2016-1285 and CVE-2016-1286 found by test suite - -* Wed Mar 9 23:00:00 2016 Tomas Hozza - 30:9.3.6-25.P1.7 -- Fix CVE-2016-1285 and CVE-2016-1286 - -* Mon Jan 18 23:00:00 2016 Tomas Hozza - 30:9.3.6-25.P1.6 -- Fix CVE-2015-8704 - -* Thu Sep 3 00:00:00 2015 Tomas Hozza - 30:9.3.6-25.P1.5 -- Fix CVE-2015-8000 - - -Dependencies Resolved - -` -) - -func TestGetChangelogCVELines(t *testing.T) { - var testsCentos6 = []struct { - in models.Package - out string - }{ - { - models.Package{ - Name: "binutils", - NewVersion: "2.20.51.0.2", - NewRelease: "5.44.el6", - }, - "", - }, - { - models.Package{ - Name: "centos-release", - NewVersion: "6", - NewRelease: "8.el6.centos.12.3", - }, - `- TESTSTRING CVE-0000-0000 -`, - }, - { - models.Package{ - Name: "dhclient", - NewVersion: "12:4.1.1", - NewRelease: "51.P1.el6.centos", - }, - `- TESTSTRING CVE-1111-1111 -`, - }, - { - models.Package{ - Name: "dhcp-common", - NewVersion: "12:4.1.1", - NewRelease: "51.P1.el6.centos", - }, - `- TESTSTRING CVE-1111-1111 -`, - }, - { - models.Package{ - Name: "coreutils-libs", - NewVersion: "8.4", - NewRelease: "43.el6", - }, - "", - }, - { - models.Package{ - Name: "file", - NewVersion: "5.04", - NewRelease: "30.el6", - }, - `- fix CVE-2014-3538 (unrestricted regular expression matching) -- fix CVE-2014-3587 (incomplete fix for CVE-2012-1571) -- fix CVE-2014-3710 (out-of-bounds read in elf note headers) -- fix CVE-2014-8116 (multiple DoS issues (resource consumption)) -- fix CVE-2014-8117 (denial of service issue (resource consumption)) -- fix CVE-2014-9620 (limit the number of ELF notes processed) -- fix CVE-2014-9653 (malformed elf file causes access to uninitialized memory) -`, - }, - { - models.Package{ - Name: "file-libs", - NewVersion: "5.04", - NewRelease: "30.el6", - }, - `- fix CVE-2014-3538 (unrestricted regular expression matching) -- fix CVE-2014-3587 (incomplete fix for CVE-2012-1571) -- fix CVE-2014-3710 (out-of-bounds read in elf note headers) -- fix CVE-2014-8116 (multiple DoS issues (resource consumption)) -- fix CVE-2014-8117 (denial of service issue (resource consumption)) -- fix CVE-2014-9620 (limit the number of ELF notes processed) -- fix CVE-2014-9653 (malformed elf file causes access to uninitialized memory) -`, - }, - } - +func TestGetDiffChangelog(t *testing.T) { r := newRedhat(config.ServerInfo{}) - r.Distro = config.Distro{ - Family: "centos", - Release: "6.7", - } - for _, tt := range testsCentos6 { - rpm2changelog, err := r.divideChangelogByPackage(stdoutCentos6) - if err != nil { - t.Errorf("err: %s", err) - } - changelog := r.getChangelogCVELines(rpm2changelog, tt.in) - if tt.out != changelog { - t.Errorf("line: expected %s, actual %s, tt: %#v", tt.out, changelog, tt) - } + type in struct { + pack models.Package + changelog string } - var testsCentos5 = []struct { - in models.Package + var tests = []struct { + in in out string }{ + // 0 { - models.Package{ - Name: "libuser", - NewVersion: "0.54.7", - NewRelease: "3.el5", + in: in{ + pack: models.Package{ + Version: "2017a", + Release: "1", + }, + changelog: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Thu Mar 2 12:00:00 2017 Patsy Franklin - 2017a-1 +- Rebase to tzdata-2017a + - Mongolia no longer observes DST. (BZ #1425222) + - Add upstream patch to fix over-runing of POSIX limit on zone abbreviations. +- Add zone1970.tab file to the install list. (BZ #1427694) + +* Wed Nov 23 12:00:00 2016 Patsy Franklin - 2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04.`, }, - "", + out: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017.`, }, + // 1 { - models.Package{ - Name: "nss_db", - NewVersion: "2.2", - NewRelease: "38.el5_11", + in: in{ + pack: models.Package{ + Version: "2004e", + Release: "2", + }, + changelog: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Wed Nov 23 12:00:00 2016 Patsy Franklin - 2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04. + +* Mon Nov 29 12:00:00 2004 Jakub Jelinek 2004g-1 +- 2004g (#141107) +- updates for Cuba + +* Mon Oct 11 12:00:00 2004 Jakub Jelinek 2004e-2 +- 2004e (#135194) +- updates for Brazil, Uruguay and Argentina`, }, - "", + out: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Wed Nov 23 12:00:00 2016 Patsy Franklin - 2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04. + +* Mon Nov 29 12:00:00 2004 Jakub Jelinek 2004g-1 +- 2004g (#141107) +- updates for Cuba`, }, + // 2 { - models.Package{ - Name: "acpid", - NewVersion: "1.0.4", - NewRelease: "82.el5", + in: in{ + pack: models.Package{ + Version: "2016j", + Release: "1", + }, + changelog: `* Mon Mar 20 12:00:00 2017 Patsy Franklin -2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Wed Nov 23 12:00:00 2016 Patsy Franklin -2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04.`, }, - "", + out: `* Mon Mar 20 12:00:00 2017 Patsy Franklin -2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017.`, }, + // 3 { - models.Package{ - Name: "mkinitrd", - NewVersion: "5.1.19.6", - NewRelease: "82.el5", + in: in{ + pack: models.Package{ + Version: "3.10.0", + Release: "327.22.1.el7", + }, + changelog: `* Thu Jun 9 21:00:00 2016 Alexander Gordeev [3.10.0-327.22.2.el7] +- [infiniband] security: Restrict use of the write() interface (Don Dutile) [1332553 1316685] {CVE-2016-4565} + +* Mon May 16 21:00:00 2016 Alexander Gordeev [3.10.0-327.22.1.el7] +- [mm] mmu_notifier: fix memory corruption (Jerome Glisse) [1335727 1307042] +- [misc] cxl: Increase timeout for detection of AFU mmio hang (Steve Best) [1335419 1329682] +- [misc] cxl: Configure the PSL for two CAPI ports on POWER8NVL (Steve Best) [1336389 1278793]`, }, - "", + out: `* Thu Jun 9 21:00:00 2016 Alexander Gordeev [3.10.0-327.22.2.el7] +- [infiniband] security: Restrict use of the write() interface (Don Dutile) [1332553 1316685] {CVE-2016-4565}`, }, + // 4 { - models.Package{ - Name: "util-linux", - NewVersion: "2.13", - NewRelease: "0.59.el5_8", + in: in{ + pack: models.Package{ + Version: "6.6.1p1", + Release: "34", + }, + + changelog: `* Wed Mar 1 21:00:00 2017 Jakub Jelen - 6.6.1p1-35 + 0.9.3-9 +- Do not send SD_NOTIFY from forked childern (#1381997) + +* Fri Feb 24 21:00:00 2017 Jakub Jelen - 6.6.1p1-34 + 0.9.3-9 +- Add SD_NOTIFY code to help systemd to track running service (#1381997)`, }, - `- fix #768382 - CVE-2011-1675 CVE-2011-1677 util-linux various flaws -`, + out: `* Wed Mar 1 21:00:00 2017 Jakub Jelen - 6.6.1p1-35 +- Do not send SD_NOTIFY from forked childern (#1381997)`, }, + // 5 { - models.Package{ - Name: "bind-libs", - NewVersion: "30:9.3.6", - NewRelease: "25.P1.el5_11.8", + in: in{ + pack: models.Package{ + Version: "2.1.23", + Release: "15.el6", + }, + changelog: `* Fri Feb 27 12:00:00 2015 Jakub Jelen 2.1.23-15.2 +- Support AIX SASL GSSAPI (#1174315) + +* Tue Nov 18 12:00:00 2014 Petr Lautrbach 2.1.23-15.1 +- check a context value in sasl_gss_encode() (#1087221) + +* Mon Jun 23 12:00:00 2014 Petr Lautrbach 2.1.23-15 +- don't use " for saslauth user's description (#1081445) +- backport the ad_compat option (#994242) +- fixed a memory leak in the client side DIGEST-MD5 code (#838628)`, }, - `- Fix issue with patch for CVE-2016-1285 and CVE-2016-1286 found by test suite -- Fix CVE-2016-1285 and CVE-2016-1286 -- Fix CVE-2015-8704 -- Fix CVE-2015-8000 -`, + out: `* Fri Feb 27 12:00:00 2015 Jakub Jelen 2.1.23-15.2 +- Support AIX SASL GSSAPI (#1174315) + +* Tue Nov 18 12:00:00 2014 Petr Lautrbach 2.1.23-15.1 +- check a context value in sasl_gss_encode() (#1087221)`, }, + // 6 { - models.Package{ - Name: "bind-utils", - NewVersion: "30:9.3.6", - NewRelease: "25.P1.el5_11.8", + in: in{ + pack: models.Package{ + Version: "3.6.20", + Release: "1.el6", + }, + changelog: `* Wed Jul 29 12:00:00 2015 Jan Stanek - 3.6.20-1.2 +- Add patch for compiler warnings highlighted by rpmdiff. + Related: rhbz#1244727 + +* Wed Jul 22 12:00:00 2015 Viktor Jancik - 3.6.20-1.el6_7.1 +- fix for CVE-2015-3416 + Resolves: #1244727 + +* Tue Nov 17 12:00:00 2009 Panu Matilainen - 3.6.20-1 +- update to 3.6.20 (http://www.sqlite.org/releaselog/3_6_20.html) + +* Tue Oct 6 12:00:00 2009 Panu Matilainen - 3.6.18-1 +- update to 3.6.18 (http://www.sqlite.org/releaselog/3_6_18.html) +- drop no longer needed test-disabler patches`, }, - `- Fix issue with patch for CVE-2016-1285 and CVE-2016-1286 found by test suite -- Fix CVE-2016-1285 and CVE-2016-1286 -- Fix CVE-2015-8704 -- Fix CVE-2015-8000 -`, + out: `* Wed Jul 29 12:00:00 2015 Jan Stanek - 3.6.20-1.2 +- Add patch for compiler warnings highlighted by rpmdiff. + Related: rhbz#1244727 + +* Wed Jul 22 12:00:00 2015 Viktor Jancik - 3.6.20-1.el6_7.1 +- fix for CVE-2015-3416 + Resolves: #1244727`, + }, + /* + // 7 + { + in: in{ + pack: models.Package{ + Version: "2:7.4.160", + Release: "1.el7", + }, + changelog: `* Mon Dec 12 21:00:00 2016 Karsten Hopp 7.4.160-1.1 + - add fix for CVE-2016-1248 + + * Wed Jan 29 21:00:00 2014 Karsten Hopp 7.4.160-1 + - patchlevel 160 + - Resolves: rhbz#1059321`, + }, + out: `* Mon Dec 12 21:00:00 2016 Karsten Hopp 7.4.160-1.1 + - add fix for CVE-2016-1248`, + }, + // 8 + { + in: in{ + pack: models.Package{ + Version: "2:1.26", + Release: "29.el7", + }, + changelog: `* Mon Jun 20 21:00:00 2016 Pavel Raiskup - 1.26-31 + - avoid double free in selinux code (rhbz#1347396) + + * Thu Jun 4 21:00:00 2015 Pavel Raiskup - 1.26-30 + - don't mistakenly set default ACLs (#1220890) + + * Fri Jan 24 21:00:00 2014 Daniel Mach - 2:1.26-29 + - Mass rebuild 2014-01-24`, + }, + out: `* Mon Jun 20 21:00:00 2016 Pavel Raiskup - 1.26-31 + - avoid double free in selinux code (rhbz#1347396) + + * Thu Jun 4 21:00:00 2015 Pavel Raiskup - 1.26-30 + - don't mistakenly set default ACLs (#1220890)`, + }, + // 9 + { + in: in{ + pack: models.Package{ + Version: "1:1.0.1e", + Release: "51.el7_2.5", + }, + changelog: `* Mon Feb 6 21:00:00 2017 Tomáš Mráz 1.0.1e-60.1 + - fix CVE-2017-3731 - DoS via truncated packets with RC4-MD5 cipher + - fix CVE-2016-8610 - DoS of single-threaded servers via excessive alerts + + * Fri Dec 4 21:00:00 2015 Tomáš Mráz 1.0.1e-52 + - fix CVE-2015-3194 - certificate verify crash with missing PSS parameter + - fix CVE-2015-3195 - X509_ATTRIBUTE memory leak + - fix CVE-2015-3196 - race condition when handling PSK identity hint + + * Tue Jun 23 21:00:00 2015 Tomáš Mráz 1.0.1e-51 + - fix the CVE-2015-1791 fix (broken server side renegotiation)`, + }, + out: `* Mon Feb 6 21:00:00 2017 Tomáš Mráz 1.0.1e-60.1 + - fix CVE-2017-3731 - DoS via truncated packets with RC4-MD5 cipher + - fix CVE-2016-8610 - DoS of single-threaded servers via excessive alerts + + * Fri Dec 4 21:00:00 2015 Tomáš Mráz 1.0.1e-52 + - fix CVE-2015-3194 - certificate verify crash with missing PSS parameter + - fix CVE-2015-3195 - X509_ATTRIBUTE memory leak + - fix CVE-2015-3196 - race condition when handling PSK identity hint`, + }, + // 10 + { + in: in{ + pack: models.Package{ + Version: "1:5.5.47", + Release: "1.el7_2", + }, + changelog: `* Wed Sep 21 21:00:00 2016 Honza Horak - 5.5.52-1 + - Rebase to 5.5.52, that also include fix for CVE-2016-6662 + Resolves: #1377974 + + * Thu Feb 18 21:00:00 2016 Jakub Dorňák - 1:5.5.47-2 + - Add warning to /usr/lib/tmpfiles.d/mariadb.conf + Resolves: #1241623 + + * Wed Feb 3 21:00:00 2016 Jakub Dorňák - 1:5.5.47-1 + - Rebase to 5.5.47 + Also fixes: CVE-2015-4792 CVE-2015-4802 CVE-2015-4815 CVE-2015-4816 + CVE-2015-4819 CVE-2015-4826 CVE-2015-4830 CVE-2015-4836 CVE-2015-4858 + CVE-2015-4861 CVE-2015-4870 CVE-2015-4879 CVE-2015-4913 CVE-2015-7744 + CVE-2016-0505 CVE-2016-0546 CVE-2016-0596 CVE-2016-0597 CVE-2016-0598 + CVE-2016-0600 CVE-2016-0606 CVE-2016-0608 CVE-2016-0609 CVE-2016-0616 + CVE-2016-2047 + Resolves: #1300621`, + }, + out: `* Wed Sep 21 21:00:00 2016 Honza Horak - 5.5.52-1 + - Rebase to 5.5.52, that also include fix for CVE-2016-6662 + Resolves: #1377974 + + * Thu Feb 18 21:00:00 2016 Jakub Dorňák - 1:5.5.47-2 + - Add warning to /usr/lib/tmpfiles.d/mariadb.conf + Resolves: #1241623`, + }, + */ + // 11 + { + in: in{ + pack: models.Package{ + Version: "0.252", + Release: "8.1.el7", + }, + changelog: `* Thu Sep 29 21:00:00 2016 Vitezslav Crhonek - 0.252-8.4 +- Remove wrong entry from usb ids. + Resolves: #1380159 + +* Mon Sep 26 21:00:00 2016 Vitezslav Crhonek - 0.252-8.3 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 + +* Tue Jun 28 21:00:00 2016 Michal Minar 0.252-8.2 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 +- Resolves: rhbz#1291614 +- Resolves: rhbz#1324198 + +* Fri Oct 23 21:00:00 2015 Michal Minar 0.252-8.1 +- Updated pci, usb and vendor ids.`, + }, + out: `* Thu Sep 29 21:00:00 2016 Vitezslav Crhonek - 0.252-8.4 +- Remove wrong entry from usb ids. + Resolves: #1380159 + +* Mon Sep 26 21:00:00 2016 Vitezslav Crhonek - 0.252-8.3 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 + +* Tue Jun 28 21:00:00 2016 Michal Minar 0.252-8.2 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 +- Resolves: rhbz#1291614 +- Resolves: rhbz#1324198`, + }, + // 12 + { + in: in{ + pack: models.Package{ + Version: "1:2.02", + Release: "0.34.el7_2", + }, + changelog: `* Mon Aug 29 21:00:00 2016 Peter Jones - 2.02-0.44 +- Work around tftp servers that don't work with multiple consecutive slashes in + file paths. + Resolves: rhbz#1217243`, + }, + out: `* Mon Aug 29 21:00:00 2016 Peter Jones - 2.02-0.44 +- Work around tftp servers that don't work with multiple consecutive slashes in + file paths. + Resolves: rhbz#1217243`, }, } - r.Distro = config.Distro{ - Family: "centos", - Release: "5.6", - } - for _, tt := range testsCentos5 { - rpm2changelog, err := r.divideChangelogByPackage(stdoutCentos5) - if err != nil { - t.Errorf("err: %s", err) - } - changelog := r.getChangelogCVELines(rpm2changelog, tt.in) - if tt.out != changelog { - t.Errorf("line: expected %s, actual %s, tt: %#v", tt.out, changelog, tt) + for i, tt := range tests { + diff, _ := r.getDiffChangelog(tt.in.pack, tt.in.changelog) + if tt.out != diff { + t.Errorf("[%d] name: expected \n%s\nactual \n%s", i, tt.out, diff) } } + +} + +func TestDivideChangelogsIntoEachPackages(t *testing.T) { + r := newRedhat(config.ServerInfo{}) + type in struct { + pack models.Package + changelog string + } + + var tests = []struct { + in string + out map[string]string + }{ + { + in: `==================== Available Packages ==================== +1:NetworkManager-1.4.0-20.el7_3.x86_64 rhui-rhel-7-server-rhui-rpms +* Mon Apr 24 21:00:00 2017 Beniamino Galvani - 1:1.4.0-20 +- vlan: use parent interface mtu as default (rh#1414186) + +* Wed Mar 29 21:00:00 2017 Beniamino Galvani - 1:1.4.0-19 +- core: alyways force a sync of the default route (rh#1431268) + + +1:NetworkManager-0.9.9.1-25.git20140326. rhui-rhel-7-server-rhui-optional-rpms +* Tue Jul 1 21:00:00 2014 Jiří Klimeš - 1:0.9.9.1-25.git20140326 +- core: fix MTU handling while merging/subtracting IP configs (rh #1093231) + +* Mon Jun 23 21:00:00 2014 Thomas Haller - 1:0.9.9.1-24.git20140326 +- core: fix crash on failure of reading bridge sysctl values (rh #1112020)`, + out: map[string]string{ + "1:NetworkManager-1.4.0-20.el7_3.x86_64": `* Mon Apr 24 21:00:00 2017 Beniamino Galvani - 1:1.4.0-20 +- vlan: use parent interface mtu as default (rh#1414186) + +* Wed Mar 29 21:00:00 2017 Beniamino Galvani - 1:1.4.0-19 +- core: alyways force a sync of the default route (rh#1431268)`, + + "1:NetworkManager-0.9.9.1-25.git20140326.": `* Tue Jul 1 21:00:00 2014 Jiří Klimeš - 1:0.9.9.1-25.git20140326 +- core: fix MTU handling while merging/subtracting IP configs (rh #1093231) + +* Mon Jun 23 21:00:00 2014 Thomas Haller - 1:0.9.9.1-24.git20140326 +- core: fix crash on failure of reading bridge sysctl values (rh #1112020)`, + }, + }, + } + + for _, tt := range tests { + changelogs := r.divideChangelogsIntoEachPackages(tt.in) + for k, v := range tt.out { + if strings.TrimSpace(v) != strings.TrimSpace(changelogs[k]) { + t.Errorf("expected: %v\nactual: %v", pp.Sprint(tt.out), pp.Sprint(changelogs)) + } + } + } + }