Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go-msfdb): add error handling and support http mode (#1308)

Browse Source 
* feat(go-msfdb): add error handling

* feat(go-msfdb): support http mode

* go get -u go-msfdb

Co-authored-by: Kota Kanbe <kotakanbe@gmail.com>
This commit is contained in:
MaineK00n
2021-09-30 11:16:41 +09:00
committed by GitHub
parent a7eb1141ae
commit 40f8272a28
3 changed files with 164 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
detector/msf.go
View File

@@ -4,9 +4,16 @@
package detector
import (
"encoding/json"
"net/http"
"time"
"github.com/cenkalti/backoff"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/logging"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
"github.com/parnurzeal/gorequest"
metasploitdb "github.com/vulsio/go-msfdb/db"
metasploitmodels "github.com/vulsio/go-msfdb/models"
"golang.org/x/xerrors"
@@ -14,37 +21,165 @@ import (
// FillWithMetasploit fills metasploit module information that has in module
func FillWithMetasploit(r *models.ScanResult, cnf config.MetasploitConf) (nMetasploitCve int, err error) {
if cnf.IsFetchViaHTTP() {
var cveIDs []string
for cveID := range r.ScannedCves {
cveIDs = append(cveIDs, cveID)
}
prefix, err := util.URLPathJoin(cnf.GetURL(), "cves")
if err != nil {
return 0, err
}
responses, err := getMetasploitsViaHTTP(cveIDs, prefix)
if err != nil {
return 0, err
}
for _, res := range responses {
msfs := []metasploitmodels.Metasploit{}
if err := json.Unmarshal([]byte(res.json), &msfs); err != nil {
return 0, err
}
metasploits := ConvertToModelsMsf(msfs)
v, ok := r.ScannedCves[res.request.cveID]
if ok {
v.Metasploits = metasploits
}
r.ScannedCves[res.request.cveID] = v
nMetasploitCve++
}
} else {
driver, locked, err := newMetasploitDB(&cnf)
if locked {
return 0, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path())
} else if err != nil {
return 0, err
}
defer func() {
if err := driver.CloseDB(); err != nil {
logging.Log.Errorf("Failed to close DB. err: %+v", err)
}
}()
driver, locked, err := newMetasploitDB(&cnf)
if locked {
return 0, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path())
} else if err != nil {
return 0, err
for cveID, vuln := range r.ScannedCves {
if cveID == "" {
continue
}
ms, err := driver.GetModuleByCveID(cveID)
if err != nil {
return 0, err
}
if len(ms) == 0 {
continue
}
modules := ConvertToModelsMsf(ms)
vuln.Metasploits = modules
r.ScannedCves[cveID] = vuln
nMetasploitCve++
}
}
defer func() {
if err := driver.CloseDB(); err != nil {
logging.Log.Errorf("Failed to close DB. err: %+v", err)
}
}()
for cveID, vuln := range r.ScannedCves {
if cveID == "" {
continue
}
ms := driver.GetModuleByCveID(cveID)
if len(ms) == 0 {
continue
}
modules := ConvertToModelsMsf(ms)
vuln.Metasploits = modules
r.ScannedCves[cveID] = vuln
nMetasploitCve++
}
return nMetasploitCve, nil
}
// ConvertToModelsMsf converts gost model to vuls model
type metasploitResponse struct {
request metasploitRequest
json string
}
func getMetasploitsViaHTTP(cveIDs []string, urlPrefix string) (
responses []metasploitResponse, err error) {
nReq := len(cveIDs)
reqChan := make(chan metasploitRequest, nReq)
resChan := make(chan metasploitResponse, nReq)
errChan := make(chan error, nReq)
defer close(reqChan)
defer close(resChan)
defer close(errChan)
go func() {
for _, cveID := range cveIDs {
reqChan <- metasploitRequest{
cveID: cveID,
}
}
}()
concurrency := 10
tasks := util.GenWorkers(concurrency)
for i := 0; i < nReq; i++ {
tasks <- func() {
req := <-reqChan
url, err := util.URLPathJoin(
urlPrefix,
req.cveID,
)
if err != nil {
errChan <- err
} else {
logging.Log.Debugf("HTTP Request to %s", url)
httpGetMetasploit(url, req, resChan, errChan)
}
}
}
timeout := time.After(2 * 60 * time.Second)
var errs []error
for i := 0; i < nReq; i++ {
select {
case res := <-resChan:
responses = append(responses, res)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return nil, xerrors.New("Timeout Fetching Metasploit")
}
}
if len(errs) != 0 {
return nil, xerrors.Errorf("Failed to fetch Metasploit. err: %w", errs)
}
return
}
type metasploitRequest struct {
cveID string
}
func httpGetMetasploit(url string, req metasploitRequest, resChan chan<- metasploitResponse, errChan chan<- error) {
var body string
var errs []error
var resp *http.Response
count, retryMax := 0, 3
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {
return nil
}
return xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %+v", url, resp, errs)
}
return nil
}
notify := func(err error, t time.Duration) {
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
errChan <- xerrors.Errorf("HTTP Error %w", err)
return
}
if count == retryMax {
errChan <- xerrors.New("Retry count exceeded")
return
}
resChan <- metasploitResponse{
request: req,
json: body,
}
}
// ConvertToModelsMsf converts metasploit model to vuls model
func ConvertToModelsMsf(ms []metasploitmodels.Metasploit) (modules []models.Metasploit) {
for _, m := range ms {
var links []string

2
go.mod
View File

@@ -53,7 +53,7 @@ require (
github.com/spf13/cobra v1.2.1
github.com/vulsio/go-cve-dictionary v0.8.1
github.com/vulsio/go-exploitdb v0.4.0
github.com/vulsio/go-msfdb v0.2.0
github.com/vulsio/go-msfdb v0.2.1-0.20210928020521-9b56a938f544
github.com/vulsio/gost v0.4.1-0.20210928234623-3e6372ba2821
github.com/vulsio/goval-dictionary v0.6.1
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect

4
go.sum
View File

@@ -1594,8 +1594,8 @@ github.com/vulsio/go-cve-dictionary v0.8.1 h1:vpxOq7OAFVqEGfxcRZ5tRKfCd8Wuioj6yN
github.com/vulsio/go-cve-dictionary v0.8.1/go.mod h1:PdkEViYpf0sx4H0YF7Sk/Xo+j8Agof4aOVoQxzL+TQA=
github.com/vulsio/go-exploitdb v0.4.0 h1:itUM3pI3FUUs20+gmwtcZsrVy0zG81yN42aHJmwAlrw=
github.com/vulsio/go-exploitdb v0.4.0/go.mod h1:C1X/lRIvDDBWDeW19Msw7asZ4q0pFjmFx/kXGns2raA=
github.com/vulsio/go-msfdb v0.2.0 h1:VY0vnQEgYB4eRwvK8yio1Tz5Yn5l7GJSh3mnUJEPceM=
github.com/vulsio/go-msfdb v0.2.0/go.mod h1:QsHhtjF4hAheLgeGJQRv/ccmE3txtOSgwzTgziyStKY=
github.com/vulsio/go-msfdb v0.2.1-0.20210928020521-9b56a938f544 h1:wG6rTODeLpm+N8wERjdVTo5kr64WqNEDR+VrKny/vAo=
github.com/vulsio/go-msfdb v0.2.1-0.20210928020521-9b56a938f544/go.mod h1:QsHhtjF4hAheLgeGJQRv/ccmE3txtOSgwzTgziyStKY=
github.com/vulsio/gost v0.4.1-0.20210928234623-3e6372ba2821 h1:MPbc8QNX9Rld5ksdWTWMdKbxfgj4qhiXosEvwfRl9Jk=
github.com/vulsio/gost v0.4.1-0.20210928234623-3e6372ba2821/go.mod h1:49trASwbe0ZhntJhEc1rv3MDGUpIhIkZktELgZ8a5YA=
github.com/vulsio/goval-dictionary v0.6.1 h1:w2AXwgPWD5/IrJ+44ywD0u5I9ILNdHvzlR+n6iu0eAQ=