Raspberry Pi OS(Raspbian) scanning using OVAL DB (#1019)
* change: never refer to ChangeLog
* change raspberry pi os use debian oval at report
* change do not use r.Family
* change gost do not use r.Family
* change use r.Family because family has a large impact
* change replace MaineK00n/goval-dictionary@raspberrypi-oval
* note Raspbian Scan Policy
* add Raspbian Changelog support policy
* change grep Package for Raspbian at fast-scan mode
* add changelog preprocessing for Raspbian
* add take note of TODO
* change Changelog fetch part to function
* change error handling
* change solve one TODO
* change make ChangelogDir once
* add comment
* fix oval support Amazon Linux :refs #824
* change to useScannedCves from ovalSupproted
* change confidence for Raspbian
* change skip package for raspbian in OVAL DB
* change separate raspbian implementation from util
* change error, log format
* change print format
* change log format(delete newline)
* change support changelog.(Debian.)gz
* Revert "change support changelog.(Debian.)gz"
This reverts commit 2265a72c67.
* change test chnage.(Debian.)gz
* change support raspbian package(*raspberry*)
* change error format
* fix regexp pattern
* fix typo
* fix changelog cache
* change rename function name
* add TestParseChangelog
* change changelog lenient match for raspbian
* fix test case
* change clog dir support symbolic link, clog save dir name append suffix
* change remove more package for raspberry pi
* fix error handling
* change module update
* change refactoring around identifying raspbian package
* update go module
* update scan image
* update scan image
* change clarify scan mode
* change raspiPackNamePattern and add test case
			
			
This commit is contained in:
		
							
								
								
									
										2
									
								
								cache/bolt.go
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										2
									
								
								cache/bolt.go
									
									
									
									
										vendored
									
									
								
							@@ -141,7 +141,7 @@ func (b Bolt) PrettyPrint(meta Meta) error {
 | 
			
		||||
	})
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// GetChangelog get the changelgo of specified packName from the Bucket
 | 
			
		||||
// GetChangelog get the changelog of specified packName from the Bucket
 | 
			
		||||
func (b Bolt) GetChangelog(servername, packName string) (changelog string, err error) {
 | 
			
		||||
	err = b.db.View(func(tx *bolt.Tx) error {
 | 
			
		||||
		bkt := tx.Bucket([]byte(servername))
 | 
			
		||||
 
 | 
			
		||||
							
								
								
									
										1
									
								
								go.sum
									
									
									
									
									
								
							
							
						
						
									
										1
									
								
								go.sum
									
									
									
									
									
								
							@@ -444,6 +444,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 | 
			
		||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 | 
			
		||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 | 
			
		||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 | 
			
		||||
github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
 | 
			
		||||
github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
 | 
			
		||||
github.com/labstack/gommon v0.2.9/go.mod h1:E8ZTmW9vw5az5/ZyHWCp0Lw4OH2ecsaBP1C/NKavGG4=
 | 
			
		||||
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
 | 
			
		||||
 
 | 
			
		||||
@@ -52,9 +52,17 @@ func (deb Debian) DetectUnfixed(driver db.DB, r *models.ScanResult, _ bool) (nCV
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// Debian Security Tracker does not support Package for Raspbian, so skip it.
 | 
			
		||||
	var scanResult models.ScanResult
 | 
			
		||||
	if r.Family != config.Raspbian {
 | 
			
		||||
		scanResult = *r
 | 
			
		||||
	} else {
 | 
			
		||||
		scanResult = r.RemoveRaspbianPackFromResult()
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	packCvesList := []packCves{}
 | 
			
		||||
	if config.Conf.Gost.IsFetchViaHTTP() {
 | 
			
		||||
		url, _ := util.URLPathJoin(config.Conf.Gost.URL, "debian", major(r.Release), "pkgs")
 | 
			
		||||
		url, _ := util.URLPathJoin(config.Conf.Gost.URL, "debian", major(scanResult.Release), "pkgs")
 | 
			
		||||
		responses, err := getAllUnfixedCvesViaHTTP(r, url)
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			return 0, err
 | 
			
		||||
@@ -79,8 +87,8 @@ func (deb Debian) DetectUnfixed(driver db.DB, r *models.ScanResult, _ bool) (nCV
 | 
			
		||||
		if driver == nil {
 | 
			
		||||
			return 0, nil
 | 
			
		||||
		}
 | 
			
		||||
		for _, pack := range r.Packages {
 | 
			
		||||
			cveDebs := driver.GetUnfixedCvesDebian(major(r.Release), pack.Name)
 | 
			
		||||
		for _, pack := range scanResult.Packages {
 | 
			
		||||
			cveDebs := driver.GetUnfixedCvesDebian(major(scanResult.Release), pack.Name)
 | 
			
		||||
			cves := []models.CveContent{}
 | 
			
		||||
			for _, cveDeb := range cveDebs {
 | 
			
		||||
				cves = append(cves, *deb.ConvertToModel(&cveDeb))
 | 
			
		||||
@@ -93,8 +101,8 @@ func (deb Debian) DetectUnfixed(driver db.DB, r *models.ScanResult, _ bool) (nCV
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		// SrcPack
 | 
			
		||||
		for _, pack := range r.SrcPackages {
 | 
			
		||||
			cveDebs := driver.GetUnfixedCvesDebian(major(r.Release), pack.Name)
 | 
			
		||||
		for _, pack := range scanResult.SrcPackages {
 | 
			
		||||
			cveDebs := driver.GetUnfixedCvesDebian(major(scanResult.Release), pack.Name)
 | 
			
		||||
			cves := []models.CveContent{}
 | 
			
		||||
			for _, cveDeb := range cveDebs {
 | 
			
		||||
				cves = append(cves, *deb.ConvertToModel(&cveDeb))
 | 
			
		||||
 
 | 
			
		||||
@@ -23,7 +23,7 @@ func NewClient(family string) Client {
 | 
			
		||||
	switch family {
 | 
			
		||||
	case cnf.RedHat, cnf.CentOS:
 | 
			
		||||
		return RedHat{}
 | 
			
		||||
	case cnf.Debian:
 | 
			
		||||
	case cnf.Debian, cnf.Raspbian:
 | 
			
		||||
		return Debian{}
 | 
			
		||||
	case cnf.Windows:
 | 
			
		||||
		return Microsoft{}
 | 
			
		||||
 
 | 
			
		||||
@@ -3,6 +3,7 @@ package models
 | 
			
		||||
import (
 | 
			
		||||
	"bytes"
 | 
			
		||||
	"fmt"
 | 
			
		||||
	"regexp"
 | 
			
		||||
	"strings"
 | 
			
		||||
 | 
			
		||||
	"golang.org/x/xerrors"
 | 
			
		||||
@@ -227,3 +228,28 @@ func (s SrcPackages) FindByBinName(name string) (*SrcPackage, bool) {
 | 
			
		||||
	}
 | 
			
		||||
	return nil, false
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// raspiPackNamePattern is a regular expression pattern to detect the Raspberry Pi specific package from the package name.
 | 
			
		||||
// e.g. libraspberrypi-dev, rpi-eeprom, python3-rpi.gpio, pi-bluetooth
 | 
			
		||||
var raspiPackNamePattern = regexp.MustCompile(`(.*raspberry.*|^rpi.*|.*-rpi.*|^pi-.*)`)
 | 
			
		||||
 | 
			
		||||
// raspiPackNamePattern is a regular expression pattern to detect the Raspberry Pi specific package from the version.
 | 
			
		||||
// e.g. ffmpeg 7:4.1.4-1+rpt7~deb10u1, vlc 3.0.10-0+deb10u1+rpt2
 | 
			
		||||
var raspiPackVersionPattern = regexp.MustCompile(`.+\+rp(t|i)\d+`)
 | 
			
		||||
 | 
			
		||||
// raspiPackNameList is a package name array of Raspberry Pi specific packages that are difficult to detect with regular expressions.
 | 
			
		||||
var raspiPackNameList = []string{"piclone", "pipanel", "pishutdown", "piwiz", "pixflat-icons"}
 | 
			
		||||
 | 
			
		||||
// IsRaspbianPackage judges whether it is a package related to Raspberry Pi from the package name and version
 | 
			
		||||
func IsRaspbianPackage(name, version string) bool {
 | 
			
		||||
	if raspiPackNamePattern.MatchString(name) || raspiPackVersionPattern.MatchString(version) {
 | 
			
		||||
		return true
 | 
			
		||||
	}
 | 
			
		||||
	for _, n := range raspiPackNameList {
 | 
			
		||||
		if n == name {
 | 
			
		||||
			return true
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	return false
 | 
			
		||||
}
 | 
			
		||||
 
 | 
			
		||||
@@ -297,3 +297,87 @@ func TestPackage_FormatVersionFromTo(t *testing.T) {
 | 
			
		||||
		})
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func Test_IsRaspbianPackage(t *testing.T) {
 | 
			
		||||
	type args struct {
 | 
			
		||||
		name string
 | 
			
		||||
		ver  string
 | 
			
		||||
	}
 | 
			
		||||
	tests := []struct {
 | 
			
		||||
		name   string
 | 
			
		||||
		in     []args
 | 
			
		||||
		expect []bool
 | 
			
		||||
	}{
 | 
			
		||||
		{
 | 
			
		||||
			name: "nameRegExp",
 | 
			
		||||
			in: []args{
 | 
			
		||||
				{
 | 
			
		||||
					name: "libraspberrypi-dev",
 | 
			
		||||
					ver:  "1.20200811-1",
 | 
			
		||||
				},
 | 
			
		||||
				{
 | 
			
		||||
					name: "rpi-eeprom",
 | 
			
		||||
					ver:  "7.10-1",
 | 
			
		||||
				},
 | 
			
		||||
				{
 | 
			
		||||
					name: "python3-rpi.gpio",
 | 
			
		||||
					ver:  "0.7.0-0.1~bpo10+1",
 | 
			
		||||
				},
 | 
			
		||||
				{
 | 
			
		||||
					name: "arping",
 | 
			
		||||
					ver:  "2.19-6",
 | 
			
		||||
				},
 | 
			
		||||
				{
 | 
			
		||||
					name: "pi-bluetooth",
 | 
			
		||||
					ver:  "0.1.14",
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expect: []bool{true, true, true, false, true, false},
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "verRegExp",
 | 
			
		||||
			in: []args{
 | 
			
		||||
				{
 | 
			
		||||
					name: "ffmpeg",
 | 
			
		||||
					ver:  "7:4.1.6-1~deb10u1+rpt1",
 | 
			
		||||
				},
 | 
			
		||||
				{
 | 
			
		||||
					name: "gcc",
 | 
			
		||||
					ver:  "4:8.3.0-1+rpi2",
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expect: []bool{true, true},
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "nameList",
 | 
			
		||||
			in: []args{
 | 
			
		||||
				{
 | 
			
		||||
					name: "piclone",
 | 
			
		||||
					ver:  "0.16",
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expect: []bool{true},
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "debianPackage",
 | 
			
		||||
			in: []args{
 | 
			
		||||
				{
 | 
			
		||||
					name: "apt",
 | 
			
		||||
					ver:  "1.8.2.1",
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expect: []bool{false},
 | 
			
		||||
		},
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	for _, tt := range tests {
 | 
			
		||||
		t.Run(tt.name, func(t *testing.T) {
 | 
			
		||||
			for i, p := range tt.in {
 | 
			
		||||
				ret := IsRaspbianPackage(p.name, p.ver)
 | 
			
		||||
				if !reflect.DeepEqual(ret, tt.expect[i]) {
 | 
			
		||||
					t.Errorf("[%s->%s] expected: %t, actual: %t, in: %#v", tt.name, tt.in[i].name, tt.expect[i], ret, tt.in[i])
 | 
			
		||||
				}
 | 
			
		||||
			}
 | 
			
		||||
		})
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 
 | 
			
		||||
@@ -472,3 +472,30 @@ type Platform struct {
 | 
			
		||||
	Name       string `json:"name"` // aws or azure or gcp or other...
 | 
			
		||||
	InstanceID string `json:"instanceID"`
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// RemoveRaspbianPackFromResult is for Raspberry Pi and removes the Raspberry Pi dedicated package from ScanResult.
 | 
			
		||||
func (r ScanResult) RemoveRaspbianPackFromResult() ScanResult {
 | 
			
		||||
	if r.Family != config.Raspbian {
 | 
			
		||||
		return r
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	result := r
 | 
			
		||||
	packs := make(Packages)
 | 
			
		||||
	for _, pack := range r.Packages {
 | 
			
		||||
		if !IsRaspbianPackage(pack.Name, pack.Version) {
 | 
			
		||||
			packs[pack.Name] = pack
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
	srcPacks := make(SrcPackages)
 | 
			
		||||
	for _, pack := range r.SrcPackages {
 | 
			
		||||
		if !IsRaspbianPackage(pack.Name, pack.Version) {
 | 
			
		||||
			srcPacks[pack.Name] = pack
 | 
			
		||||
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	result.Packages = packs
 | 
			
		||||
	result.SrcPackages = srcPacks
 | 
			
		||||
 | 
			
		||||
	return result
 | 
			
		||||
}
 | 
			
		||||
 
 | 
			
		||||
@@ -38,7 +38,13 @@ func (o DebianBase) update(r *models.ScanResult, defPacks defPacks) {
 | 
			
		||||
				defPacks.def.Debian.CveID)
 | 
			
		||||
			cveContents = models.CveContents{}
 | 
			
		||||
		}
 | 
			
		||||
		vinfo.Confidences.AppendIfMissing(models.OvalMatch)
 | 
			
		||||
		if r.Family != config.Raspbian {
 | 
			
		||||
			vinfo.Confidences.AppendIfMissing(models.OvalMatch)
 | 
			
		||||
		} else {
 | 
			
		||||
			if len(vinfo.Confidences) == 0 {
 | 
			
		||||
				vinfo.Confidences.AppendIfMissing(models.OvalMatch)
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
		cveContents[ctype] = ovalContent
 | 
			
		||||
		vinfo.CveContents = cveContents
 | 
			
		||||
	}
 | 
			
		||||
@@ -132,12 +138,28 @@ func (o Debian) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err
 | 
			
		||||
 | 
			
		||||
	var relatedDefs ovalResult
 | 
			
		||||
	if config.Conf.OvalDict.IsFetchViaHTTP() {
 | 
			
		||||
		if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil {
 | 
			
		||||
			return 0, err
 | 
			
		||||
		if r.Family != config.Raspbian {
 | 
			
		||||
			if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil {
 | 
			
		||||
				return 0, err
 | 
			
		||||
			}
 | 
			
		||||
		} else {
 | 
			
		||||
			// OVAL does not support Package for Raspbian, so skip it.
 | 
			
		||||
			result := r.RemoveRaspbianPackFromResult()
 | 
			
		||||
			if relatedDefs, err = getDefsByPackNameViaHTTP(&result); err != nil {
 | 
			
		||||
				return 0, err
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
	} else {
 | 
			
		||||
		if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil {
 | 
			
		||||
			return 0, err
 | 
			
		||||
		if r.Family != config.Raspbian {
 | 
			
		||||
			if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil {
 | 
			
		||||
				return 0, err
 | 
			
		||||
			}
 | 
			
		||||
		} else {
 | 
			
		||||
			// OVAL does not support Package for Raspbian, so skip it.
 | 
			
		||||
			result := r.RemoveRaspbianPackFromResult()
 | 
			
		||||
			if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, &result); err != nil {
 | 
			
		||||
				return 0, err
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
 
 | 
			
		||||
@@ -327,7 +327,8 @@ func isOvalDefAffected(def ovalmodels.Definition, req request, family string, ru
 | 
			
		||||
				config.Amazon,
 | 
			
		||||
				config.SUSEEnterpriseServer,
 | 
			
		||||
				config.Debian,
 | 
			
		||||
				config.Ubuntu:
 | 
			
		||||
				config.Ubuntu,
 | 
			
		||||
				config.Raspbian:
 | 
			
		||||
				// Use fixed state in OVAL for these distros.
 | 
			
		||||
				return true, false, ovalPack.Version
 | 
			
		||||
			}
 | 
			
		||||
@@ -362,7 +363,8 @@ var esVerPattern = regexp.MustCompile(`\.el(\d+)(?:_\d+)?`)
 | 
			
		||||
func lessThan(family, newVer string, packInOVAL ovalmodels.Package) (bool, error) {
 | 
			
		||||
	switch family {
 | 
			
		||||
	case config.Debian,
 | 
			
		||||
		config.Ubuntu:
 | 
			
		||||
		config.Ubuntu,
 | 
			
		||||
		config.Raspbian:
 | 
			
		||||
		vera, err := debver.NewVersion(newVer)
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			return false, err
 | 
			
		||||
 
 | 
			
		||||
@@ -49,7 +49,7 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode
 | 
			
		||||
	wpVulnCaches := map[string]string{}
 | 
			
		||||
	for _, r := range rs {
 | 
			
		||||
		if c.Conf.RefreshCve || needToRefreshCve(r) {
 | 
			
		||||
			if ovalSupported(&r) {
 | 
			
		||||
			if !useScannedCves(&r) {
 | 
			
		||||
				r.ScannedCves = models.VulnInfos{}
 | 
			
		||||
			}
 | 
			
		||||
			cpeURIs := []string{}
 | 
			
		||||
@@ -285,7 +285,7 @@ func FillWithOval(driver ovaldb.DB, r *models.ScanResult) (nCVEs int, err error)
 | 
			
		||||
	var ovalFamily string
 | 
			
		||||
 | 
			
		||||
	switch r.Family {
 | 
			
		||||
	case c.Debian:
 | 
			
		||||
	case c.Debian, c.Raspbian:
 | 
			
		||||
		ovalClient = oval.NewDebian()
 | 
			
		||||
		ovalFamily = c.Debian
 | 
			
		||||
	case c.Ubuntu:
 | 
			
		||||
@@ -311,7 +311,7 @@ func FillWithOval(driver ovaldb.DB, r *models.ScanResult) (nCVEs int, err error)
 | 
			
		||||
	case c.Amazon:
 | 
			
		||||
		ovalClient = oval.NewAmazon()
 | 
			
		||||
		ovalFamily = c.Amazon
 | 
			
		||||
	case c.Raspbian, c.FreeBSD, c.Windows:
 | 
			
		||||
	case c.FreeBSD, c.Windows:
 | 
			
		||||
		return 0, nil
 | 
			
		||||
	case c.ServerTypePseudo:
 | 
			
		||||
		return 0, nil
 | 
			
		||||
 
 | 
			
		||||
@@ -388,15 +388,14 @@ func formatChangelogs(r models.ScanResult) string {
 | 
			
		||||
	}
 | 
			
		||||
	return strings.Join(buf, "\n")
 | 
			
		||||
}
 | 
			
		||||
func ovalSupported(r *models.ScanResult) bool {
 | 
			
		||||
func useScannedCves(r *models.ScanResult) bool {
 | 
			
		||||
	switch r.Family {
 | 
			
		||||
	case
 | 
			
		||||
		config.Amazon,
 | 
			
		||||
		config.FreeBSD,
 | 
			
		||||
		config.Raspbian:
 | 
			
		||||
		return false
 | 
			
		||||
		return true
 | 
			
		||||
	}
 | 
			
		||||
	return true
 | 
			
		||||
	return false
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func needToRefreshCve(r models.ScanResult) bool {
 | 
			
		||||
 
 | 
			
		||||
							
								
								
									
										171
									
								
								scan/debian.go
									
									
									
									
									
								
							
							
						
						
									
										171
									
								
								scan/debian.go
									
									
									
									
									
								
							@@ -2,6 +2,8 @@ package scan
 | 
			
		||||
 | 
			
		||||
import (
 | 
			
		||||
	"bufio"
 | 
			
		||||
	"crypto/rand"
 | 
			
		||||
	"encoding/binary"
 | 
			
		||||
	"fmt"
 | 
			
		||||
	"regexp"
 | 
			
		||||
	"strconv"
 | 
			
		||||
@@ -305,7 +307,18 @@ func (o *debian) scanPackages() error {
 | 
			
		||||
		return nil
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	if o.getServerInfo().Mode.IsDeep() || o.Distro.Family == config.Raspbian {
 | 
			
		||||
	if !o.getServerInfo().Mode.IsDeep() && o.Distro.Family == config.Raspbian {
 | 
			
		||||
		raspbianPacks := o.grepRaspbianPackages(updatable)
 | 
			
		||||
		unsecures, err := o.scanUnsecurePackages(raspbianPacks)
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			o.log.Errorf("Failed to scan vulnerable packages: %s", err)
 | 
			
		||||
			return err
 | 
			
		||||
		}
 | 
			
		||||
		o.VulnInfos = unsecures
 | 
			
		||||
		return nil
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	if o.getServerInfo().Mode.IsDeep() {
 | 
			
		||||
		unsecures, err := o.scanUnsecurePackages(updatable)
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			o.log.Errorf("Failed to scan vulnerable packages: %s", err)
 | 
			
		||||
@@ -314,6 +327,7 @@ func (o *debian) scanPackages() error {
 | 
			
		||||
		o.VulnInfos = unsecures
 | 
			
		||||
		return nil
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	return nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
@@ -463,6 +477,17 @@ func (o *debian) aptGetUpdate() error {
 | 
			
		||||
	return nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) grepRaspbianPackages(updatables models.Packages) models.Packages {
 | 
			
		||||
	raspbianPacks := models.Packages{}
 | 
			
		||||
 | 
			
		||||
	for _, pack := range updatables {
 | 
			
		||||
		if models.IsRaspbianPackage(pack.Name, pack.Version) {
 | 
			
		||||
			raspbianPacks[pack.Name] = pack
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
	return raspbianPacks
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) {
 | 
			
		||||
	// Setup changelog cache
 | 
			
		||||
	current := cache.Meta{
 | 
			
		||||
@@ -477,12 +502,29 @@ func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInf
 | 
			
		||||
		return nil, err
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// Make a directory for saving changelog to get changelog in Raspbian
 | 
			
		||||
	tmpClogPath := ""
 | 
			
		||||
	if o.Distro.Family == config.Raspbian {
 | 
			
		||||
		tmpClogPath, err = o.makeTempChangelogDir()
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			return nil, err
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// Collect CVE information of upgradable packages
 | 
			
		||||
	vulnInfos, err := o.scanChangelogs(updatable, meta)
 | 
			
		||||
	vulnInfos, err := o.scanChangelogs(updatable, meta, tmpClogPath)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, xerrors.Errorf("Failed to scan unsecure packages. err: %s", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// Delete a directory for saving changelog to get changelog in Raspbian
 | 
			
		||||
	if o.Distro.Family == config.Raspbian {
 | 
			
		||||
		err := o.deleteTempChangelogDir(tmpClogPath)
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			return nil, xerrors.Errorf("Failed to delete directory to save changelog for Raspbian. err: %s", err)
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	return vulnInfos, nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
@@ -601,6 +643,39 @@ func (o *debian) parseAptGetUpgrade(stdout string) (updatableNames []string, err
 | 
			
		||||
	return
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) makeTempChangelogDir() (string, error) {
 | 
			
		||||
	suffix, err := generateSuffix()
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return "", err
 | 
			
		||||
	}
 | 
			
		||||
	path := "/tmp/vuls-" + suffix
 | 
			
		||||
	cmd := fmt.Sprintf(`mkdir -p %s`, path)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r := o.exec(cmd, noSudo)
 | 
			
		||||
	if !r.isSuccess() {
 | 
			
		||||
		return "", xerrors.Errorf("Failed to create directory to save changelog for Raspbian. cmd: %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr)
 | 
			
		||||
	}
 | 
			
		||||
	return path, nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func generateSuffix() (string, error) {
 | 
			
		||||
	var n uint64
 | 
			
		||||
	if err := binary.Read(rand.Reader, binary.LittleEndian, &n); err != nil {
 | 
			
		||||
		return "", xerrors.Errorf("Failed to generate Suffix. err: %s", err)
 | 
			
		||||
	}
 | 
			
		||||
	return strconv.FormatUint(n, 36), nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) deleteTempChangelogDir(tmpClogPath string) error {
 | 
			
		||||
	cmd := fmt.Sprintf(`rm -rf %s`, tmpClogPath)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r := o.exec(cmd, noSudo)
 | 
			
		||||
	if !r.isSuccess() {
 | 
			
		||||
		return xerrors.Errorf("Failed to delete directory to save changelog for Raspbian. cmd: %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr)
 | 
			
		||||
	}
 | 
			
		||||
	return nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// DetectedCveID has CveID, Confidence and DetectionMethod fields
 | 
			
		||||
// LenientMatching will be true if this vulnerability is not detected by accurate version matching.
 | 
			
		||||
// see https://github.com/future-architect/vuls/pull/328
 | 
			
		||||
@@ -609,7 +684,7 @@ type DetectedCveID struct {
 | 
			
		||||
	Confidence models.Confidence
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) scanChangelogs(updatablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) {
 | 
			
		||||
func (o *debian) scanChangelogs(updatablePacks models.Packages, meta *cache.Meta, tmpClogPath string) (models.VulnInfos, error) {
 | 
			
		||||
	type response struct {
 | 
			
		||||
		pack           *models.Package
 | 
			
		||||
		DetectedCveIDs []DetectedCveID
 | 
			
		||||
@@ -645,7 +720,7 @@ func (o *debian) scanChangelogs(updatablePacks models.Packages, meta *cache.Meta
 | 
			
		||||
					// if the changelog is not in cache or failed to get from local cache,
 | 
			
		||||
					// get the changelog of the package via internet.
 | 
			
		||||
					// After that, store it in the cache.
 | 
			
		||||
					if cveIDs, pack, err := o.fetchParseChangelog(p); err != nil {
 | 
			
		||||
					if cveIDs, pack, err := o.fetchParseChangelog(p, tmpClogPath); err != nil {
 | 
			
		||||
						errChan <- err
 | 
			
		||||
					} else {
 | 
			
		||||
						resChan <- response{pack, cveIDs}
 | 
			
		||||
@@ -743,13 +818,22 @@ func (o *debian) getChangelogCache(meta *cache.Meta, pack models.Package) string
 | 
			
		||||
	return changelog
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) fetchParseChangelog(pack models.Package) ([]DetectedCveID, *models.Package, error) {
 | 
			
		||||
func (o *debian) fetchParseChangelog(pack models.Package, tmpClogPath string) ([]DetectedCveID, *models.Package, error) {
 | 
			
		||||
	cmd := ""
 | 
			
		||||
 | 
			
		||||
	switch o.Distro.Family {
 | 
			
		||||
	case config.Ubuntu, config.Raspbian:
 | 
			
		||||
	case config.Ubuntu:
 | 
			
		||||
		cmd = fmt.Sprintf(`PAGER=cat apt-get -q=2 changelog %s`, pack.Name)
 | 
			
		||||
	case config.Debian:
 | 
			
		||||
		cmd = fmt.Sprintf(`PAGER=cat aptitude -q=2 changelog %s`, pack.Name)
 | 
			
		||||
	case config.Raspbian:
 | 
			
		||||
		changelogPath, err := o.getChangelogPath(pack.Name, tmpClogPath)
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			// Ignore this Error.
 | 
			
		||||
			o.log.Warnf("Failed to get Path to Changelog for Package: %s, err: %s", pack.Name, err)
 | 
			
		||||
			return nil, nil, nil
 | 
			
		||||
		}
 | 
			
		||||
		cmd = fmt.Sprintf(`gzip -cd %s | cat`, changelogPath)
 | 
			
		||||
	}
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
 | 
			
		||||
@@ -765,7 +849,7 @@ func (o *debian) fetchParseChangelog(pack models.Package) ([]DetectedCveID, *mod
 | 
			
		||||
 | 
			
		||||
	if clogFilledPack.Changelog.Method != models.FailedToGetChangelog {
 | 
			
		||||
		err := cache.DB.PutChangelog(
 | 
			
		||||
			o.getServerInfo().GetServerName(), pack.Name, pack.Changelog.Contents)
 | 
			
		||||
			o.getServerInfo().GetServerName(), pack.Name, stdout)
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			return nil, nil, xerrors.New("Failed to put changelog into cache")
 | 
			
		||||
		}
 | 
			
		||||
@@ -775,6 +859,64 @@ func (o *debian) fetchParseChangelog(pack models.Package) ([]DetectedCveID, *mod
 | 
			
		||||
	return cveIDs, clogFilledPack, nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) getChangelogPath(packName, tmpClogPath string) (string, error) {
 | 
			
		||||
	// `apt download` downloads deb package to current directory
 | 
			
		||||
	cmd := fmt.Sprintf(`cd %s && apt download %s`, tmpClogPath, packName)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r := o.exec(cmd, noSudo)
 | 
			
		||||
	if !r.isSuccess() {
 | 
			
		||||
		return "", xerrors.Errorf("Failed to Fetch deb package. cmd: %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	cmd = fmt.Sprintf(`find %s -name "%s_*.deb"`, tmpClogPath, packName)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r = o.exec(cmd, noSudo)
 | 
			
		||||
	if !r.isSuccess() || r.Stdout == "" {
 | 
			
		||||
		return "", xerrors.Errorf("Failed to find deb package. cmd: %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// e.g. <tmpPath>/ffmpeg_7%3a4.1.6-1~deb10u1+rpt1_armhf.deb\n => <tmpPath>/ffmpeg_7%3a4.1.6-1~deb10u1+rpt1_armhf
 | 
			
		||||
	packChangelogDir := strings.Split(r.Stdout, ".deb")[0]
 | 
			
		||||
	cmd = fmt.Sprintf(`dpkg-deb -x %s.deb %s`, packChangelogDir, packChangelogDir)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r = o.exec(cmd, noSudo)
 | 
			
		||||
	if !r.isSuccess() {
 | 
			
		||||
		return "", xerrors.Errorf("Failed to dpkg-deb. cmd: %s, status: %d, stdout: %s, stderr: %s", cmd, r.ExitStatus, r.Stdout, r.Stderr)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// recurse if doc/packName is symbolic link
 | 
			
		||||
	changelogDocDir := fmt.Sprintf("%s/usr/share/doc/%s", packChangelogDir, packName)
 | 
			
		||||
	cmd = fmt.Sprintf(`test -L %s && readlink --no-newline %s`, changelogDocDir, changelogDocDir)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r = o.exec(cmd, noSudo)
 | 
			
		||||
	if r.isSuccess() {
 | 
			
		||||
		return o.getChangelogPath(r.Stdout, tmpClogPath)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	var results = make(map[string]execResult, 2)
 | 
			
		||||
	packChangelogPath := fmt.Sprintf("%s/changelog.Debian.gz", changelogDocDir)
 | 
			
		||||
	cmd = fmt.Sprintf(`test -e %s`, packChangelogPath)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r = o.exec(cmd, noSudo)
 | 
			
		||||
	if r.isSuccess() {
 | 
			
		||||
		return packChangelogPath, nil
 | 
			
		||||
	}
 | 
			
		||||
	results["changelog.Debian.gz"] = r
 | 
			
		||||
 | 
			
		||||
	packChangelogPath = fmt.Sprintf("%s/changelog.gz", changelogDocDir)
 | 
			
		||||
	cmd = fmt.Sprintf(`test -e %s`, packChangelogPath)
 | 
			
		||||
	cmd = util.PrependProxyEnv(cmd)
 | 
			
		||||
	r = o.exec(cmd, noSudo)
 | 
			
		||||
	if r.isSuccess() {
 | 
			
		||||
		return packChangelogPath, nil
 | 
			
		||||
	}
 | 
			
		||||
	results["changelog.gz"] = r
 | 
			
		||||
 | 
			
		||||
	return "", xerrors.Errorf(
 | 
			
		||||
		"Failed to get changelog.\nresult(changelog.Debian.gz):%v\nresult(changelog.Debian.gz):%v",
 | 
			
		||||
		results["changelog.Debian.gz"], results["changelog.gz"])
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (o *debian) getCveIDsFromChangelog(
 | 
			
		||||
	changelog, name, ver string) ([]DetectedCveID, *models.Package) {
 | 
			
		||||
 | 
			
		||||
@@ -874,6 +1016,21 @@ func (o *debian) parseChangelog(changelog, name, ver string, confidence models.C
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	if !found {
 | 
			
		||||
		if o.Distro.Family == config.Raspbian {
 | 
			
		||||
			pack := o.Packages[name]
 | 
			
		||||
			pack.Changelog = models.Changelog{
 | 
			
		||||
				Contents: strings.Join(buf, "\n"),
 | 
			
		||||
				Method:   models.ChangelogLenientMatchStr,
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			cves := []DetectedCveID{}
 | 
			
		||||
			for _, id := range cveIDs {
 | 
			
		||||
				cves = append(cves, DetectedCveID{id, confidence})
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			return cves, &pack, nil
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		pack := o.Packages[name]
 | 
			
		||||
		pack.Changelog = models.Changelog{
 | 
			
		||||
			Contents: "",
 | 
			
		||||
 
 | 
			
		||||
@@ -746,3 +746,121 @@ libuuid1:amd64: /lib/x86_64-linux-gnu/libuuid.so.1.3.0`,
 | 
			
		||||
		})
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func TestParseChangelog(t *testing.T) {
 | 
			
		||||
	type args struct {
 | 
			
		||||
		changelog string
 | 
			
		||||
		name      string
 | 
			
		||||
		ver       string
 | 
			
		||||
	}
 | 
			
		||||
	type expect struct {
 | 
			
		||||
		cveIDs []DetectedCveID
 | 
			
		||||
		pack   models.Package
 | 
			
		||||
	}
 | 
			
		||||
	tests := []struct {
 | 
			
		||||
		packName string
 | 
			
		||||
		args     args
 | 
			
		||||
		expect   expect
 | 
			
		||||
	}{
 | 
			
		||||
		{
 | 
			
		||||
			packName: "vlc",
 | 
			
		||||
			args: args{
 | 
			
		||||
				changelog: `vlc (3.0.11-0+deb10u1+rpt2) buster; urgency=medium
 | 
			
		||||
 | 
			
		||||
  * Add MMAL patch 19
 | 
			
		||||
 | 
			
		||||
 -- Serge Schneider <serge@raspberrypi.com>  Wed, 29 Jul 2020 14:28:28 +0100
 | 
			
		||||
 | 
			
		||||
vlc (3.0.11-0+deb10u1+rpt1) buster; urgency=high
 | 
			
		||||
 | 
			
		||||
  * Add MMAL patch 18
 | 
			
		||||
  * Add libxrandr-dev dependency
 | 
			
		||||
  * Add libdrm-dev dependency
 | 
			
		||||
  * Disable vdpau, libva, aom
 | 
			
		||||
  * Enable dav1d
 | 
			
		||||
 | 
			
		||||
 -- Serge Schneider <serge@raspberrypi.com>  Wed, 17 Jun 2020 10:30:58 +0100
 | 
			
		||||
 | 
			
		||||
vlc (3.0.11-0+deb10u1) buster-security; urgency=high
 | 
			
		||||
 | 
			
		||||
  * New upstream release
 | 
			
		||||
    - Fix heap-based buffer overflow in hxxx_nall (CVE-2020-13428)
 | 
			
		||||
 | 
			
		||||
 -- Sebastian Ramacher <sramacher@debian.org>  Mon, 15 Jun 2020 23:08:37 +0200
 | 
			
		||||
 | 
			
		||||
vlc (3.0.10-0+deb10u1) buster-security; urgency=medium`,
 | 
			
		||||
				name: "vlc",
 | 
			
		||||
				ver:  "3.0.10-0+deb10u1+rpt2",
 | 
			
		||||
			},
 | 
			
		||||
			expect: expect{
 | 
			
		||||
				cveIDs: []DetectedCveID{{"CVE-2020-13428", models.ChangelogExactMatch}},
 | 
			
		||||
				pack: models.Package{Changelog: models.Changelog{
 | 
			
		||||
					Contents: `vlc (3.0.11-0+deb10u1+rpt2) buster; urgency=medium
 | 
			
		||||
 | 
			
		||||
  * Add MMAL patch 19
 | 
			
		||||
 | 
			
		||||
 -- Serge Schneider <serge@raspberrypi.com>  Wed, 29 Jul 2020 14:28:28 +0100
 | 
			
		||||
 | 
			
		||||
vlc (3.0.11-0+deb10u1+rpt1) buster; urgency=high
 | 
			
		||||
 | 
			
		||||
  * Add MMAL patch 18
 | 
			
		||||
  * Add libxrandr-dev dependency
 | 
			
		||||
  * Add libdrm-dev dependency
 | 
			
		||||
  * Disable vdpau, libva, aom
 | 
			
		||||
  * Enable dav1d
 | 
			
		||||
 | 
			
		||||
 -- Serge Schneider <serge@raspberrypi.com>  Wed, 17 Jun 2020 10:30:58 +0100
 | 
			
		||||
 | 
			
		||||
vlc (3.0.11-0+deb10u1) buster-security; urgency=high
 | 
			
		||||
 | 
			
		||||
  * New upstream release
 | 
			
		||||
    - Fix heap-based buffer overflow in hxxx_nall (CVE-2020-13428)
 | 
			
		||||
 | 
			
		||||
 -- Sebastian Ramacher <sramacher@debian.org>  Mon, 15 Jun 2020 23:08:37 +0200
 | 
			
		||||
`,
 | 
			
		||||
					Method: models.ChangelogExactMatchStr,
 | 
			
		||||
				}},
 | 
			
		||||
			},
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			packName: "realvnc-vnc-server",
 | 
			
		||||
			args: args{
 | 
			
		||||
				changelog: `realvnc-vnc (6.7.2.42622) stable; urgency=low
 | 
			
		||||
 | 
			
		||||
  * Debian package for VNC Server
 | 
			
		||||
 | 
			
		||||
 -- RealVNC <noreply@realvnc.com>  Wed, 13 May 2020 19:51:40 +0100
 | 
			
		||||
 | 
			
		||||
`,
 | 
			
		||||
				name: "realvnc-vnc-server",
 | 
			
		||||
				ver:  "6.7.1.42348",
 | 
			
		||||
			},
 | 
			
		||||
			expect: expect{
 | 
			
		||||
				cveIDs: []DetectedCveID{},
 | 
			
		||||
				pack: models.Package{Changelog: models.Changelog{
 | 
			
		||||
					Contents: `realvnc-vnc (6.7.2.42622) stable; urgency=low
 | 
			
		||||
 | 
			
		||||
  * Debian package for VNC Server
 | 
			
		||||
 | 
			
		||||
 -- RealVNC <noreply@realvnc.com>  Wed, 13 May 2020 19:51:40 +0100
 | 
			
		||||
`,
 | 
			
		||||
					Method: models.ChangelogLenientMatchStr,
 | 
			
		||||
				}},
 | 
			
		||||
			},
 | 
			
		||||
		},
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	o := newDebian(config.ServerInfo{})
 | 
			
		||||
	o.Distro = config.Distro{Family: config.Raspbian}
 | 
			
		||||
	for _, tt := range tests {
 | 
			
		||||
		t.Run(tt.packName, func(t *testing.T) {
 | 
			
		||||
			cveIDs, pack, _ := o.parseChangelog(tt.args.changelog, tt.args.name, tt.args.ver, models.ChangelogExactMatch)
 | 
			
		||||
			if !reflect.DeepEqual(cveIDs, tt.expect.cveIDs) {
 | 
			
		||||
				t.Errorf("[%s]->cveIDs: expected: %s, actual: %s", tt.packName, tt.expect.cveIDs, cveIDs)
 | 
			
		||||
			}
 | 
			
		||||
			if !reflect.DeepEqual(pack.Changelog.Contents, tt.expect.pack.Changelog.Contents) {
 | 
			
		||||
				t.Errorf("[%s]->changelog.Contents: expected: %s, actual: %s", tt.packName, tt.expect.pack.Changelog.Contents, pack.Changelog.Contents)
 | 
			
		||||
			}
 | 
			
		||||
		})
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 
 | 
			
		||||
		Reference in New Issue
	
	Block a user