v0.5.0 (no backwards compatibility) (#478)

* Change config.toml, Auto-generate UUIDs, change structure of optional field * Detect processes affected by update using yum-ps (#482) Detect processes affected by update using yum-ps * Detect processes needs restart using checkrestart on Debian and Ubuntu. * pass cpename by args when calling FillCveInfo (#513) * fix new db (#502) * Include Version,Revision in JSON * Include hostname in JSON * Update goval-dictionary's commit hash in Gopkg.lock * Remove README.ja.md * update packages (#596) * fix: change ControlPath to .vuls of SSH option (#618) * feat: checkrestart for Ubuntu and Debian (#622) * feat: checkrestart for Ubuntu and Debian * fix: dependencies check logic of configtest * feat: need-restarting on RedHat * refactor: Process.ProcName to Process.Name * feat: detect a systemd service name of need-restarting-process * feat: detect a systemd service name of need-restarting-process on Ubuntu * feat: fill a service name of need-restarting-process, init-system * Support NVD JSON and CVSS3 of JVN (#605) * fix: compile errors * fix: Show CVSS3 on TUI * fix: test cases * fix: Avoid null in JSON * Fix maxCvssScore (#621) * Fix maxCvssScore * Update vulninfos.go * fix(init): remove unnecessary log initialization * refactor(nvd): use only json feed if exists json data. if not, use xml feed * fix(scan): make Confidence slice * feat(CWE): Display CWE name to TUI * feat(cwe): import CWE defs in Japanese * feat(cwe): add OWASP Top 10 ranking to CWE if applicable * feat(scan): add -fast-root mode, implement scan/amazon.go * refactor(const): change const name JVN to Jvn * feat(scan): add -fast-root mode, implement scan/centos.go * refactor(dep): update deps * fix(amazon): deps check * feat(scan): add -fast-root mode, implement scan/rhel.go * feat(scan): add -fast-root mode, implement scan/oracle.go * fix complile err * feat(scan): add -fast-root mode, implement scan/debian.go * fix testcase * fix(amazon): scan using yum * fix(configtest): change error message, status when no scannnable servers * Fix(scan): detect init process logic * fix(tui): display cvss as table format * fix(scan): parse a output of reboot-notifier on CentOS6.9 * fix(tui): don't display score, vector when score is zero * fix(scan): add -offline mode to suse scanner * fix(scan): fix help message * feat(scan): enable to define scan mode for each servers in config.toml #510 * refactor(config): chagne cpeNames to cpeURIs * refactor(config): change dependencyCheckXMLPath to owaspDCXMLPath * fix(config): containers -> containersIncluded, Excluded, containerType * feature(report): enable to define cpeURIs for each contaner * feature(report): enable to specify owasp dc xml path for each container * fix(discover): fix a template displayed at the end of discover * feature(report): add ignorePkgsRegexp #665 * feature(report): enable to define ignoreCves for each container #666 * fix(report): Displayed nothing in TUI detail area when CweID is nil * Gopkg.toml diet * feat(server): support server mode (#678) * feat(server): support server mode * Lock go version * Use the latest kernel release among the installed release when the running kernel release is unknown * Add TestViaHTTP * Set logger to go-cve-dictionary client * Add -to-localfile * Add -to-http option to report * Load -to-http conf from config.toml * Support gost (#676) * feat(gost): Support RedHat API * feat(gost): Support Debian Security Tracker * feat(db): display error msg when SQLite3 is locked at the beginning of reporting. * feat(gost): TUI * Only use RedHat information of installed packages * feat(tui): show mitigation on TUI * feat(gost): support redis backend * fix test case * fix nil pointer when db is nil * fix(gost): detect vulns of src packages for Debian * feat(gost): implement redis backend for gost redhat api * feat(report): display fixState of unfixed pkgs * fix(report): display distincted cweIDs * feat(slack): display gost info * feat(slack): display mitigation * feat(report): display available patch state as fixed/total * fix(tui): display - if source of reference is empty * update deps * fix(report): key in ScanResult JSON be lowerCamelcase. * some keys to lower camel * fix(configtest): dep check logic of yum-plugin-ps * fix(tui): format * feat(report): add -format-list option * fix(report): -format-full-text * fix(report): report -format-full-text * fix(report): display v3 score detected by gost * fix(scan): scan in fast mode if not defined in config.toml * fix(gost): fetch RedHat data for fixed CVEs * feat(report): show number of cves detected in each database * fix(report): show new version as `Unknown` in offline and fast scan mode * fix(report): fix num of upadtable and fixed * fix(report): set `Not fixed yet` if packageStatus is empty * refact(gost): make convertToModel public * fix(test): fix test case * update deps * fix(report): include gost score in MaxCvssScore * [WIP] feat(config): enable to set options in config.toml instead of cmd opt (#690) * feat(config): enable to set options in config.toml instead of cmd opt * fix(config): change Conf.Report.Slack to Conf.Slack * fix(discover): change tempalte * fix(report): fix config.toml auto-generate with -uuid * Add endpoint for health check and change endpoint * refact(cmd): refactor flag set * fix(report): enable to specify opts with cmd arg and env value * fix(scan): enable to parse the release version of amazon linux 2 * add(report) add -to-saas option (#695) * add(report) add -to-saas option * ignore other writer if -to-saas * fix(saas) fix bug * fix(scan): need-restarting needs internet connection * fix(scan,configtest): check scan mode * refactor(scan): change func name * fix(suse): support offline mode, bug fix on AWS, zypper --no-color * fix(tui): fix nil pointer when no vulns in tui * feat(report): enable to define CPE FS format in config.toml * fix(vet): fix warnings of go vet * fix(travis): go version to 1.11 * update deps
2018-08-27 13:51:09 +09:00
parent d785fc2a54
commit 44fa2c5800
82 changed files with 14019 additions and 2485 deletions
--- a/gost/util.go
+++ b/gost/util.go
@@ -0,0 +1,201 @@
+/* Vuls - Vulnerability Scanner
+Copyright (C) 2016  Future Architect, Inc. Japan.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gost
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/cenkalti/backoff"
+	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+	"github.com/parnurzeal/gorequest"
+)
+
+type response struct {
+	request request
+	json    string
+}
+
+func getCvesViaHTTP(cveIDs []string, urlPrefix string) (
+	responses []response, err error) {
+	nReq := len(cveIDs)
+	reqChan := make(chan request, nReq)
+	resChan := make(chan response, nReq)
+	errChan := make(chan error, nReq)
+	defer close(reqChan)
+	defer close(resChan)
+	defer close(errChan)
+
+	go func() {
+		for _, cveID := range cveIDs {
+			reqChan <- request{
+				cveID: cveID,
+			}
+		}
+	}()
+
+	concurrency := 10
+	tasks := util.GenWorkers(concurrency)
+	for i := 0; i < nReq; i++ {
+		tasks <- func() {
+			select {
+			case req := <-reqChan:
+				url, err := util.URLPathJoin(
+					urlPrefix,
+					req.cveID,
+				)
+				if err != nil {
+					errChan <- err
+				} else {
+					util.Log.Debugf("HTTP Request to %s", url)
+					httpGet(url, req, resChan, errChan)
+				}
+			}
+		}
+	}
+
+	timeout := time.After(2 * 60 * time.Second)
+	var errs []error
+	for i := 0; i < nReq; i++ {
+		select {
+		case res := <-resChan:
+			responses = append(responses, res)
+		case err := <-errChan:
+			errs = append(errs, err)
+		case <-timeout:
+			return nil, fmt.Errorf("Timeout Fetching OVAL")
+		}
+	}
+	if len(errs) != 0 {
+		return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
+	}
+	return
+}
+
+type request struct {
+	osMajorVersion string
+	packName       string
+	isSrcPack      bool
+	cveID          string
+}
+
+func getAllUnfixedCvesViaHTTP(r *models.ScanResult, urlPrefix string) (
+	responses []response, err error) {
+
+	nReq := len(r.Packages) + len(r.SrcPackages)
+	reqChan := make(chan request, nReq)
+	resChan := make(chan response, nReq)
+	errChan := make(chan error, nReq)
+	defer close(reqChan)
+	defer close(resChan)
+	defer close(errChan)
+
+	go func() {
+		for _, pack := range r.Packages {
+			reqChan <- request{
+				osMajorVersion: major(r.Release),
+				packName:       pack.Name,
+				isSrcPack:      false,
+			}
+		}
+		for _, pack := range r.SrcPackages {
+			reqChan <- request{
+				osMajorVersion: major(r.Release),
+				packName:       pack.Name,
+				isSrcPack:      true,
+			}
+		}
+	}()
+
+	concurrency := 10
+	tasks := util.GenWorkers(concurrency)
+	for i := 0; i < nReq; i++ {
+		tasks <- func() {
+			select {
+			case req := <-reqChan:
+				url, err := util.URLPathJoin(
+					urlPrefix,
+					req.packName,
+					"unfixed-cves",
+				)
+				if err != nil {
+					errChan <- err
+				} else {
+					util.Log.Debugf("HTTP Request to %s", url)
+					httpGet(url, req, resChan, errChan)
+				}
+			}
+		}
+	}
+
+	timeout := time.After(2 * 60 * time.Second)
+	var errs []error
+	for i := 0; i < nReq; i++ {
+		select {
+		case res := <-resChan:
+			responses = append(responses, res)
+		case err := <-errChan:
+			errs = append(errs, err)
+		case <-timeout:
+			return nil, fmt.Errorf("Timeout Fetching OVAL")
+		}
+	}
+	if len(errs) != 0 {
+		return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
+	}
+	return
+}
+
+func httpGet(url string, req request, resChan chan<- response, errChan chan<- error) {
+	var body string
+	var errs []error
+	var resp *http.Response
+	count, retryMax := 0, 3
+	f := func() (err error) {
+		//  resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
+		resp, body, errs = gorequest.New().Get(url).End()
+		if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
+			count++
+			if count == retryMax {
+				return nil
+			}
+			return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
+				errs, url, resp)
+		}
+		return nil
+	}
+	notify := func(err error, t time.Duration) {
+		util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
+	}
+	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
+	if err != nil {
+		errChan <- fmt.Errorf("HTTP Error %s", err)
+		return
+	}
+	if count == retryMax {
+		errChan <- fmt.Errorf("HRetry count exceeded")
+		return
+	}
+
+	resChan <- response{
+		request: req,
+		json:    body,
+	}
+}