Support scanning Ubuntu using Gost (#1243)

* chore: add vuls binary in gitignore * feat(gost): support ubuntu * chore(debian): fix typo * feat(ubuntu): more detail on CveContent * chore: update .gitignore * chore: update gost deps * feat(ubuntu): add test in gost/ubuntu * chore: fix typo * Revert "chore: fix typo" This reverts commit 9f2f1db233. * docs: update README
2021-07-08 08:31:46 +09:00
parent 0bf12412d6
commit 0b9ec05181
9 changed files with 350 additions and 8 deletions
--- a/README.md
+++ b/README.md
@@ -71,6 +71,7 @@ Vuls is a tool created to solve the problems listed above. It has the following
  - [Alpine-secdb](https://git.alpinelinux.org/cgit/alpine-secdb/)
  - [Red Hat Security Advisories](https://access.redhat.com/security/security-updates/)
  - [Debian Security Bug Tracker](https://security-tracker.debian.org/tracker/)
+  - [Ubuntu CVE Tracker](https://people.canonical.com/~ubuntu-security/cve/)

 - Commands(yum, zypper, pkg-audit)
  - RHSA / ALAS / ELSA / FreeBSD-SA
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
 	github.com/aws/aws-sdk-go v1.36.31
 	github.com/boltdb/bolt v1.3.1
-	github.com/briandowns/spinner v1.15.0 // indirect
+	github.com/briandowns/spinner v1.16.0 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/cheggaaa/pb/v3 v3.0.8 // indirect
 	github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b
@@ -33,7 +33,7 @@ require (
 	github.com/knqyf263/go-cpe v0.0.0-20201213041631-54f6ab28673f
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936
-	github.com/knqyf263/gost v0.1.11-0.20210615205949-22120a6441d8
+	github.com/knqyf263/gost v0.2.0
 	github.com/kotakanbe/go-cve-dictionary v0.15.14
 	github.com/kotakanbe/go-pingscanner v0.1.0
 	github.com/kotakanbe/goval-dictionary v0.3.6-0.20210625044258-9be85404d7dd
--- a/go.sum
+++ b/go.sum
@@ -257,8 +257,8 @@ github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2
 github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/briandowns/spinner v1.12.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
-github.com/briandowns/spinner v1.15.0 h1:L0jR0MYN7OAeMwpTzDZWIeqyDLXtTeJFxqoq+sL0VQM=
-github.com/briandowns/spinner v1.15.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
+github.com/briandowns/spinner v1.16.0 h1:DFmp6hEaIx2QXXuqSJmtfSBSAjRmpGiKG6ip2Wm/yOs=
+github.com/briandowns/spinner v1.16.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
@@ -921,8 +921,8 @@ github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936 h1:HDjRqot
 github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936/go.mod h1:i4sF0l1fFnY1aiw08QQSwVAFxHEm311Me3WsU/X7nL0=
 github.com/knqyf263/go-rpmdb v0.0.0-20201215100354-a9e3110d8ee1 h1:sRDvjjWoHLWAxtPXBKYRJp8Ot4ugxYE/ZyADl3jzc1g=
 github.com/knqyf263/go-rpmdb v0.0.0-20201215100354-a9e3110d8ee1/go.mod h1:RDPNeIkU5NWXtt0OMEoILyxwUC/DyXeRtK295wpqSi0=
-github.com/knqyf263/gost v0.1.11-0.20210615205949-22120a6441d8 h1:kpIwDZ5QHns0kmfACWlrDUPP/KFMIec/3pn8uVy5v/A=
-github.com/knqyf263/gost v0.1.11-0.20210615205949-22120a6441d8/go.mod h1:VgcvOkxaqKbf695UyQrNFKMPQgWChNlGUnJbw8kmET8=
+github.com/knqyf263/gost v0.2.0 h1:A62Kzj7+0T5/TdAAwDH6uK5HfffN+6h/wClX5lTSzZ8=
+github.com/knqyf263/gost v0.2.0/go.mod h1:VgcvOkxaqKbf695UyQrNFKMPQgWChNlGUnJbw8kmET8=
 github.com/knqyf263/nested v0.0.1 h1:Sv26CegUMhjt19zqbBKntjwESdxe5hxVPSk0+AKjdUc=
 github.com/knqyf263/nested v0.0.1/go.mod h1:zwhsIhMkBg90DTOJQvxPkKIypEHPYkgWHs4gybdlUmk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
--- a/gost/debian.go
+++ b/gost/debian.go
@@ -42,7 +42,7 @@ func (deb Debian) DetectCVEs(r *models.ScanResult, _ bool) (nCVEs int, err error
 		return 0, nil
 	}

-	// Add linux and set the version of running kernel to search OVAL.
+	// Add linux and set the version of running kernel to search Gost.
 	if r.Container.ContainerID == "" {
 		newVer := ""
 		if p, ok := r.Packages["linux-image-"+r.RunningKernel.Release]; ok {
--- a/gost/gost.go
+++ b/gost/gost.go
@@ -69,6 +69,8 @@ func NewClient(cnf config.GostConf, family string) (Client, error) {
 		return RedHat{Base{DBDriver: driver}}, nil
 	case constant.Debian, constant.Raspbian:
 		return Debian{Base{DBDriver: driver}}, nil
+	case constant.Ubuntu:
+		return Ubuntu{Base{DBDriver: driver}}, nil
 	case constant.Windows:
 		return Microsoft{Base{DBDriver: driver}}, nil
 	default:
--- a/gost/ubuntu.go
+++ b/gost/ubuntu.go
@@ -0,0 +1,190 @@
+// +build !scanner
+
+package gost
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/future-architect/vuls/logging"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+	gostmodels "github.com/knqyf263/gost/models"
+)
+
+// Ubuntu is Gost client for Ubuntu
+type Ubuntu struct {
+	Base
+}
+
+func (ubu Ubuntu) supported(version string) bool {
+	_, ok := map[string]string{
+		"1404": "trusty",
+		"1604": "xenial",
+		"1804": "bionic",
+		"2004": "focal",
+		"2010": "groovy",
+		"2104": "hirsute",
+	}[version]
+	return ok
+}
+
+// DetectCVEs fills cve information that has in Gost
+func (ubu Ubuntu) DetectCVEs(r *models.ScanResult, _ bool) (nCVEs int, err error) {
+	ubuReleaseVer := strings.Replace(r.Release, ".", "", 1)
+	if !ubu.supported(ubuReleaseVer) {
+		logging.Log.Warnf("Ubuntu %s is not supported yet", r.Release)
+		return 0, nil
+	}
+
+	linuxImage := "linux-image-" + r.RunningKernel.Release
+	// Add linux and set the version of running kernel to search Gost.
+	if r.Container.ContainerID == "" {
+		newVer := ""
+		if p, ok := r.Packages[linuxImage]; ok {
+			newVer = p.NewVersion
+		}
+		r.Packages["linux"] = models.Package{
+			Name:       "linux",
+			Version:    r.RunningKernel.Version,
+			NewVersion: newVer,
+		}
+	}
+
+	packCvesList := []packCves{}
+	if ubu.DBDriver.Cnf.IsFetchViaHTTP() {
+		url, _ := util.URLPathJoin(ubu.DBDriver.Cnf.GetURL(), "ubuntu", ubuReleaseVer, "pkg")
+		responses, err := getAllUnfixedCvesViaHTTP(r, url)
+		if err != nil {
+			return 0, err
+		}
+
+		for _, res := range responses {
+			ubuCves := map[string]gostmodels.UbuntuCVE{}
+			if err := json.Unmarshal([]byte(res.json), &ubuCves); err != nil {
+				return 0, err
+			}
+			cves := []models.CveContent{}
+			for _, ubucve := range ubuCves {
+				cves = append(cves, *ubu.ConvertToModel(&ubucve))
+			}
+			packCvesList = append(packCvesList, packCves{
+				packName:  res.request.packName,
+				isSrcPack: res.request.isSrcPack,
+				cves:      cves,
+			})
+		}
+	} else {
+		if ubu.DBDriver.DB == nil {
+			return 0, nil
+		}
+		for _, pack := range r.Packages {
+			ubuCves := ubu.DBDriver.DB.GetUnfixedCvesUbuntu(ubuReleaseVer, pack.Name)
+			cves := []models.CveContent{}
+			for _, ubucve := range ubuCves {
+				cves = append(cves, *ubu.ConvertToModel(&ubucve))
+			}
+			packCvesList = append(packCvesList, packCves{
+				packName:  pack.Name,
+				isSrcPack: false,
+				cves:      cves,
+			})
+		}
+
+		// SrcPack
+		for _, pack := range r.SrcPackages {
+			ubuCves := ubu.DBDriver.DB.GetUnfixedCvesUbuntu(ubuReleaseVer, pack.Name)
+			cves := []models.CveContent{}
+			for _, ubucve := range ubuCves {
+				cves = append(cves, *ubu.ConvertToModel(&ubucve))
+			}
+			packCvesList = append(packCvesList, packCves{
+				packName:  pack.Name,
+				isSrcPack: true,
+				cves:      cves,
+			})
+		}
+	}
+
+	delete(r.Packages, "linux")
+
+	for _, p := range packCvesList {
+		for _, cve := range p.cves {
+			v, ok := r.ScannedCves[cve.CveID]
+			if ok {
+				if v.CveContents == nil {
+					v.CveContents = models.NewCveContents(cve)
+				} else {
+					v.CveContents[models.UbuntuAPI] = cve
+				}
+			} else {
+				v = models.VulnInfo{
+					CveID:       cve.CveID,
+					CveContents: models.NewCveContents(cve),
+					Confidences: models.Confidences{models.UbuntuAPIMatch},
+				}
+				nCVEs++
+			}
+
+			names := []string{}
+			if p.isSrcPack {
+				if srcPack, ok := r.SrcPackages[p.packName]; ok {
+					for _, binName := range srcPack.BinaryNames {
+						if _, ok := r.Packages[binName]; ok {
+							names = append(names, binName)
+						}
+					}
+				}
+			} else {
+				if p.packName == "linux" {
+					names = append(names, linuxImage)
+				} else {
+					names = append(names, p.packName)
+				}
+			}
+
+			for _, name := range names {
+				v.AffectedPackages = v.AffectedPackages.Store(models.PackageFixStatus{
+					Name:        name,
+					FixState:    "open",
+					NotFixedYet: true,
+				})
+			}
+			r.ScannedCves[cve.CveID] = v
+		}
+	}
+	return nCVEs, nil
+}
+
+// ConvertToModel converts gost model to vuls model
+func (ubu Ubuntu) ConvertToModel(cve *gostmodels.UbuntuCVE) *models.CveContent {
+	references := []models.Reference{}
+	for _, r := range cve.References {
+		if strings.Contains(r.Reference, "https://cve.mitre.org/cgi-bin/cvename.cgi?name=") {
+			references = append(references, models.Reference{Source: "CVE", Link: r.Reference})
+		} else {
+			references = append(references, models.Reference{Link: r.Reference})
+		}
+	}
+
+	for _, b := range cve.Bugs {
+		references = append(references, models.Reference{Source: "Bug", Link: b.Bug})
+	}
+
+	for _, u := range cve.Upstreams {
+		for _, upstreamLink := range u.UpstreamLinks {
+			references = append(references, models.Reference{Source: "UPSTREAM", Link: upstreamLink.Link})
+		}
+	}
+
+	return &models.CveContent{
+		Type:          models.UbuntuAPI,
+		CveID:         cve.Candidate,
+		Summary:       cve.Description,
+		Cvss2Severity: cve.Priority,
+		Cvss3Severity: cve.Priority,
+		SourceLink:    "https://ubuntu.com/security/" + cve.Candidate,
+		References:    references,
+		Published:     cve.PublicDate,
+	}
+}
--- a/gost/ubuntu_test.go
+++ b/gost/ubuntu_test.go
@@ -0,0 +1,137 @@
+package gost
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/future-architect/vuls/models"
+	gostmodels "github.com/knqyf263/gost/models"
+)
+
+func TestUbuntu_Supported(t *testing.T) {
+	type args struct {
+		ubuReleaseVer string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "14.04 is supported",
+			args: args{
+				ubuReleaseVer: "1404",
+			},
+			want: true,
+		},
+		{
+			name: "16.04 is supported",
+			args: args{
+				ubuReleaseVer: "1604",
+			},
+			want: true,
+		},
+		{
+			name: "18.04 is supported",
+			args: args{
+				ubuReleaseVer: "1804",
+			},
+			want: true,
+		},
+		{
+			name: "20.04 is supported",
+			args: args{
+				ubuReleaseVer: "2004",
+			},
+			want: true,
+		},
+		{
+			name: "20.10 is supported",
+			args: args{
+				ubuReleaseVer: "2010",
+			},
+			want: true,
+		},
+		{
+			name: "21.04 is supported",
+			args: args{
+				ubuReleaseVer: "2104",
+			},
+			want: true,
+		},
+		{
+			name: "empty string is not supported yet",
+			args: args{
+				ubuReleaseVer: "",
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ubu := Ubuntu{}
+			if got := ubu.supported(tt.args.ubuReleaseVer); got != tt.want {
+				t.Errorf("Ubuntu.Supported() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestUbuntuConvertToModel(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    gostmodels.UbuntuCVE
+		expected models.CveContent
+	}{
+		{
+			name: "gost Ubuntu.ConvertToModel",
+			input: gostmodels.UbuntuCVE{
+				Candidate:  "CVE-2021-3517",
+				PublicDate: time.Date(2021, 5, 19, 14, 15, 0, 0, time.UTC),
+				References: []gostmodels.UbuntuReference{
+					{Reference: "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-3517"},
+					{Reference: "https://gitlab.gnome.org/GNOME/libxml2/-/issues/235"},
+					{Reference: "https://gitlab.gnome.org/GNOME/libxml2/-/commit/bf22713507fe1fc3a2c4b525cf0a88c2dc87a3a2"}},
+				Description: "description.",
+				Notes:       []gostmodels.UbuntuNote{},
+				Bugs:        []gostmodels.UbuntuBug{{Bug: "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=987738"}},
+				Priority:    "medium",
+				Patches: []gostmodels.UbuntuPatch{
+					{PackageName: "libxml2", ReleasePatches: []gostmodels.UbuntuReleasePatch{
+						{ReleaseName: "focal", Status: "needed", Note: ""},
+					}},
+				},
+				Upstreams: []gostmodels.UbuntuUpstream{{
+					PackageName: "libxml2", UpstreamLinks: []gostmodels.UbuntuUpstreamLink{
+						{Link: "https://gitlab.gnome.org/GNOME/libxml2/-/commit/50f06b3efb638efb0abd95dc62dca05ae67882c2"},
+					},
+				}},
+			},
+			expected: models.CveContent{
+				Type:          models.UbuntuAPI,
+				CveID:         "CVE-2021-3517",
+				Summary:       "description.",
+				Cvss2Severity: "medium",
+				Cvss3Severity: "medium",
+				SourceLink:    "https://ubuntu.com/security/CVE-2021-3517",
+				References: []models.Reference{
+					{Source: "CVE", Link: "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-3517"},
+					{Link: "https://gitlab.gnome.org/GNOME/libxml2/-/issues/235"},
+					{Link: "https://gitlab.gnome.org/GNOME/libxml2/-/commit/bf22713507fe1fc3a2c4b525cf0a88c2dc87a3a2"},
+					{Source: "Bug", Link: "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=987738"},
+					{Source: "UPSTREAM", Link: "https://gitlab.gnome.org/GNOME/libxml2/-/commit/50f06b3efb638efb0abd95dc62dca05ae67882c2"}},
+				Published: time.Date(2021, 5, 19, 14, 15, 0, 0, time.UTC),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ubu := Ubuntu{}
+			got := ubu.ConvertToModel(&tt.input)
+			if !reflect.DeepEqual(got, &tt.expected) {
+				t.Errorf("Ubuntu.ConvertToModel() = %#v, want %#v", got, &tt.expected)
+			}
+		})
+	}
+}
--- a/models/cvecontents.go
+++ b/models/cvecontents.go
@@ -245,6 +245,8 @@ func NewCveContentType(name string) CveContentType {
 		return RedHatAPI
 	case "debian_security_tracker":
 		return DebianSecurityTracker
+	case "ubuntu_api":
+		return UbuntuAPI
 	case "microsoft":
 		return Microsoft
 	case "wordpress":
@@ -282,6 +284,9 @@ const (
 	// Ubuntu is Ubuntu
 	Ubuntu CveContentType = "ubuntu"

+	// UbuntuAPI is Ubuntu
+	UbuntuAPI CveContentType = "ubuntu_api"
+
 	// Oracle is Oracle Linux
 	Oracle CveContentType = "oracle"

@@ -317,10 +322,11 @@ var AllCveContetTypes = CveContentTypes{
 	RedHat,
 	RedHatAPI,
 	Debian,
+	DebianSecurityTracker,
 	Ubuntu,
+	UbuntuAPI,
 	Amazon,
 	SUSE,
-	DebianSecurityTracker,
 	WpScan,
 	Trivy,
 	GitHub,
--- a/models/vulninfos.go
+++ b/models/vulninfos.go
@@ -826,6 +826,9 @@ const (
 	// DebianSecurityTrackerMatchStr is a String representation of DebianSecurityTrackerMatch
 	DebianSecurityTrackerMatchStr = "DebianSecurityTrackerMatch"

+	// UbuntuAPIMatchStr is a String representation of UbuntuAPIMatch
+	UbuntuAPIMatchStr = "UbuntuAPIMatch"
+
 	// TrivyMatchStr is a String representation of Trivy
 	TrivyMatchStr = "TrivyMatch"

@@ -867,6 +870,9 @@ var (
 	// DebianSecurityTrackerMatch ranking how confident the CVE-ID was detected correctly
 	DebianSecurityTrackerMatch = Confidence{100, DebianSecurityTrackerMatchStr, 0}

+	// UbuntuAPIMatch ranking how confident the CVE-ID was detected correctly
+	UbuntuAPIMatch = Confidence{100, UbuntuAPIMatchStr, 0}
+
 	// TrivyMatch ranking how confident the CVE-ID was detected correctly
 	TrivyMatch = Confidence{100, TrivyMatchStr, 0}