From 40f8272a28f6127983e5b39eb2e140fd761aeeff Mon Sep 17 00:00:00 2001 From: MaineK00n Date: Thu, 30 Sep 2021 11:16:41 +0900 Subject: [PATCH] feat(go-msfdb): add error handling and support http mode (#1308) * feat(go-msfdb): add error handling * feat(go-msfdb): support http mode * go get -u go-msfdb Co-authored-by: Kota Kanbe --- detector/msf.go | 187 +++++++++++++++++++++++++++++++++++++++++------- go.mod | 2 +- go.sum | 4 +- 3 files changed, 164 insertions(+), 29 deletions(-) diff --git a/detector/msf.go b/detector/msf.go index 8017b5b8..8252c2a5 100644 --- a/detector/msf.go +++ b/detector/msf.go @@ -4,9 +4,16 @@ package detector import ( + "encoding/json" + "net/http" + "time" + + "github.com/cenkalti/backoff" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/logging" "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + "github.com/parnurzeal/gorequest" metasploitdb "github.com/vulsio/go-msfdb/db" metasploitmodels "github.com/vulsio/go-msfdb/models" "golang.org/x/xerrors" @@ -14,37 +21,165 @@ import ( // FillWithMetasploit fills metasploit module information that has in module func FillWithMetasploit(r *models.ScanResult, cnf config.MetasploitConf) (nMetasploitCve int, err error) { + if cnf.IsFetchViaHTTP() { + var cveIDs []string + for cveID := range r.ScannedCves { + cveIDs = append(cveIDs, cveID) + } + prefix, err := util.URLPathJoin(cnf.GetURL(), "cves") + if err != nil { + return 0, err + } + responses, err := getMetasploitsViaHTTP(cveIDs, prefix) + if err != nil { + return 0, err + } + for _, res := range responses { + msfs := []metasploitmodels.Metasploit{} + if err := json.Unmarshal([]byte(res.json), &msfs); err != nil { + return 0, err + } + metasploits := ConvertToModelsMsf(msfs) + v, ok := r.ScannedCves[res.request.cveID] + if ok { + v.Metasploits = metasploits + } + r.ScannedCves[res.request.cveID] = v + nMetasploitCve++ + } + } else { + driver, locked, err := newMetasploitDB(&cnf) + if locked { + return 0, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path()) + } else if err != nil { + return 0, err + } + defer func() { + if err := driver.CloseDB(); err != nil { + logging.Log.Errorf("Failed to close DB. err: %+v", err) + } + }() - driver, locked, err := newMetasploitDB(&cnf) - if locked { - return 0, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path()) - } else if err != nil { - return 0, err + for cveID, vuln := range r.ScannedCves { + if cveID == "" { + continue + } + ms, err := driver.GetModuleByCveID(cveID) + if err != nil { + return 0, err + } + if len(ms) == 0 { + continue + } + modules := ConvertToModelsMsf(ms) + vuln.Metasploits = modules + r.ScannedCves[cveID] = vuln + nMetasploitCve++ + } } - defer func() { - if err := driver.CloseDB(); err != nil { - logging.Log.Errorf("Failed to close DB. err: %+v", err) - } - }() - - for cveID, vuln := range r.ScannedCves { - if cveID == "" { - continue - } - ms := driver.GetModuleByCveID(cveID) - if len(ms) == 0 { - continue - } - modules := ConvertToModelsMsf(ms) - vuln.Metasploits = modules - r.ScannedCves[cveID] = vuln - nMetasploitCve++ - } - return nMetasploitCve, nil } -// ConvertToModelsMsf converts gost model to vuls model +type metasploitResponse struct { + request metasploitRequest + json string +} + +func getMetasploitsViaHTTP(cveIDs []string, urlPrefix string) ( + responses []metasploitResponse, err error) { + nReq := len(cveIDs) + reqChan := make(chan metasploitRequest, nReq) + resChan := make(chan metasploitResponse, nReq) + errChan := make(chan error, nReq) + defer close(reqChan) + defer close(resChan) + defer close(errChan) + + go func() { + for _, cveID := range cveIDs { + reqChan <- metasploitRequest{ + cveID: cveID, + } + } + }() + + concurrency := 10 + tasks := util.GenWorkers(concurrency) + for i := 0; i < nReq; i++ { + tasks <- func() { + req := <-reqChan + url, err := util.URLPathJoin( + urlPrefix, + req.cveID, + ) + if err != nil { + errChan <- err + } else { + logging.Log.Debugf("HTTP Request to %s", url) + httpGetMetasploit(url, req, resChan, errChan) + } + } + } + + timeout := time.After(2 * 60 * time.Second) + var errs []error + for i := 0; i < nReq; i++ { + select { + case res := <-resChan: + responses = append(responses, res) + case err := <-errChan: + errs = append(errs, err) + case <-timeout: + return nil, xerrors.New("Timeout Fetching Metasploit") + } + } + if len(errs) != 0 { + return nil, xerrors.Errorf("Failed to fetch Metasploit. err: %w", errs) + } + return +} + +type metasploitRequest struct { + cveID string +} + +func httpGetMetasploit(url string, req metasploitRequest, resChan chan<- metasploitResponse, errChan chan<- error) { + var body string + var errs []error + var resp *http.Response + count, retryMax := 0, 3 + f := func() (err error) { + // resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() + resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + count++ + if count == retryMax { + return nil + } + return xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %+v", url, resp, errs) + } + return nil + } + notify := func(err error, t time.Duration) { + logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err) + } + err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) + if err != nil { + errChan <- xerrors.Errorf("HTTP Error %w", err) + return + } + if count == retryMax { + errChan <- xerrors.New("Retry count exceeded") + return + } + + resChan <- metasploitResponse{ + request: req, + json: body, + } +} + +// ConvertToModelsMsf converts metasploit model to vuls model func ConvertToModelsMsf(ms []metasploitmodels.Metasploit) (modules []models.Metasploit) { for _, m := range ms { var links []string diff --git a/go.mod b/go.mod index 3ff2cb00..d0922300 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/spf13/cobra v1.2.1 github.com/vulsio/go-cve-dictionary v0.8.1 github.com/vulsio/go-exploitdb v0.4.0 - github.com/vulsio/go-msfdb v0.2.0 + github.com/vulsio/go-msfdb v0.2.1-0.20210928020521-9b56a938f544 github.com/vulsio/gost v0.4.1-0.20210928234623-3e6372ba2821 github.com/vulsio/goval-dictionary v0.6.1 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect diff --git a/go.sum b/go.sum index 0d9047f5..9ff671b5 100644 --- a/go.sum +++ b/go.sum @@ -1594,8 +1594,8 @@ github.com/vulsio/go-cve-dictionary v0.8.1 h1:vpxOq7OAFVqEGfxcRZ5tRKfCd8Wuioj6yN github.com/vulsio/go-cve-dictionary v0.8.1/go.mod h1:PdkEViYpf0sx4H0YF7Sk/Xo+j8Agof4aOVoQxzL+TQA= github.com/vulsio/go-exploitdb v0.4.0 h1:itUM3pI3FUUs20+gmwtcZsrVy0zG81yN42aHJmwAlrw= github.com/vulsio/go-exploitdb v0.4.0/go.mod h1:C1X/lRIvDDBWDeW19Msw7asZ4q0pFjmFx/kXGns2raA= -github.com/vulsio/go-msfdb v0.2.0 h1:VY0vnQEgYB4eRwvK8yio1Tz5Yn5l7GJSh3mnUJEPceM= -github.com/vulsio/go-msfdb v0.2.0/go.mod h1:QsHhtjF4hAheLgeGJQRv/ccmE3txtOSgwzTgziyStKY= +github.com/vulsio/go-msfdb v0.2.1-0.20210928020521-9b56a938f544 h1:wG6rTODeLpm+N8wERjdVTo5kr64WqNEDR+VrKny/vAo= +github.com/vulsio/go-msfdb v0.2.1-0.20210928020521-9b56a938f544/go.mod h1:QsHhtjF4hAheLgeGJQRv/ccmE3txtOSgwzTgziyStKY= github.com/vulsio/gost v0.4.1-0.20210928234623-3e6372ba2821 h1:MPbc8QNX9Rld5ksdWTWMdKbxfgj4qhiXosEvwfRl9Jk= github.com/vulsio/gost v0.4.1-0.20210928234623-3e6372ba2821/go.mod h1:49trASwbe0ZhntJhEc1rv3MDGUpIhIkZktELgZ8a5YA= github.com/vulsio/goval-dictionary v0.6.1 h1:w2AXwgPWD5/IrJ+44ywD0u5I9ILNdHvzlR+n6iu0eAQ=