Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(report): set http timeout 10 sec (#1154)

Browse Source 
* fix(report): set http timeout 10 sec

* fix: add an error handling
This commit is contained in:
Kota Kanbe
2021-01-30 09:40:33 +09:00
committed by GitHub
parent 39b19444fe
commit 4dcbd865cc
17 changed files with 105 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config/exploitconf.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/parnurzeal/gorequest"
"golang.org/x/xerrors"
@@ -63,7 +64,7 @@ func (cnf *ExploitConf) CheckHTTPHealth() error {
}
url := fmt.Sprintf("%s/health", cnf.URL)
resp, _, errs := gorequest.New().Get(url).End()
resp, _, errs := gorequest.New().Timeout(10 * time.Second).Get(url).End()
// resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
// resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {

3
config/gocvedictconf.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/parnurzeal/gorequest"
"golang.org/x/xerrors"
@@ -63,7 +64,7 @@ func (cnf *GoCveDictConf) CheckHTTPHealth() error {
}
url := fmt.Sprintf("%s/health", cnf.URL)
resp, _, errs := gorequest.New().SetDebug(Conf.Debug).Get(url).End()
resp, _, errs := gorequest.New().Timeout(10 * time.Second).SetDebug(Conf.Debug).Get(url).End()
// resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return xerrors.Errorf("Failed to request to CVE server. url: %s, errs: %s",

3
config/gostconf.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/parnurzeal/gorequest"
"golang.org/x/xerrors"
@@ -63,7 +64,7 @@ func (cnf *GostConf) CheckHTTPHealth() error {
}
url := fmt.Sprintf("%s/health", cnf.URL)
resp, _, errs := gorequest.New().Get(url).End()
resp, _, errs := gorequest.New().Timeout(10 * time.Second).Get(url).End()
// resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
// resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {

3
config/govaldictconf.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/parnurzeal/gorequest"
"golang.org/x/xerrors"
@@ -64,7 +65,7 @@ func (cnf *GovalDictConf) CheckHTTPHealth() error {
}
url := fmt.Sprintf("%s/health", cnf.URL)
resp, _, errs := gorequest.New().Get(url).End()
resp, _, errs := gorequest.New().Timeout(10 * time.Second).Get(url).End()
// resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
// resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {

2
exploit/util.go
View File

@@ -85,7 +85,7 @@ func httpGet(url string, req request, resChan chan<- response, errChan chan<- er
count, retryMax := 0, 3
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Get(url).End()
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {

5
github/github.go
View File

@@ -22,6 +22,7 @@ func DetectGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string)
src := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
//TODO Proxy
httpClient := oauth2.NewClient(context.Background(), src)
// TODO Use `https://github.com/shurcooL/githubv4` if the tool supports vulnerabilityAlerts Endpoint
@@ -32,10 +33,12 @@ func DetectGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string)
for {
jsonStr := fmt.Sprintf(jsonfmt, owner, repo, 100, after)
req, err := http.NewRequest("POST",
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
"https://api.github.com/graphql",
bytes.NewBuffer([]byte(jsonStr)),
)
defer cancel()
if err != nil {
return 0, err
}

2
gost/util.go
View File

@@ -154,7 +154,7 @@ func httpGet(url string, req request, resChan chan<- response, errChan chan<- er
count, retryMax := 0, 3
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Get(url).End()
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {

1
models/vulninfos.go
View File

@@ -530,6 +530,7 @@ func (c Cvss) Format() string {
return fmt.Sprintf("%3.1f/%s %s", c.Score, c.Vector, c.Severity)
}
// SeverityToCvssScoreRange returns CVSS score range
func (c Cvss) SeverityToCvssScoreRange() string {
return severityToCvssScoreRange(c.Severity)
}

4
oval/oval.go
View File

@@ -39,7 +39,7 @@ func (b Base) CheckIfOvalFetched(driver db.DB, osFamily, release string) (fetche
}
url, _ := util.URLPathJoin(cnf.Conf.OvalDict.URL, "count", osFamily, release)
resp, body, errs := gorequest.New().Get(url).End()
resp, body, errs := gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return false, xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %s", url, resp, errs)
}
@@ -57,7 +57,7 @@ func (b Base) CheckIfOvalFresh(driver db.DB, osFamily, release string) (ok bool,
lastModified = driver.GetLastModified(osFamily, release)
} else {
url, _ := util.URLPathJoin(cnf.Conf.OvalDict.URL, "lastmodified", osFamily, release)
resp, body, errs := gorequest.New().Get(url).End()
resp, body, errs := gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return false, xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %s", url, resp, errs)
}

2
oval/util.go
View File

@@ -189,7 +189,7 @@ func httpGet(url string, req request, resChan chan<- response, errChan chan<- er
count, retryMax := 0, 3
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Get(url).End()
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {

26
report/chatwork.go
View File

@@ -1,14 +1,17 @@
package report
import (
"context"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
)
// ChatWorkWriter send report to ChatWork
@@ -48,26 +51,25 @@ func (w ChatWorkWriter) Write(rs ...models.ScanResult) (err error) {
func chatWorkpostMessage(room, token, message string) error {
uri := fmt.Sprintf("https://api.chatwork.com/v2/rooms/%s/messages=%s", room, token)
payload := url.Values{"body": {message}}
payload := url.Values{
"body": {message},
}
reqs, err := http.NewRequest("POST", uri, strings.NewReader(payload.Encode()))
reqs.Header.Add("X-ChatWorkToken", token)
reqs.Header.Add("Content-Type", "application/x-www-form-urlencoded")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, strings.NewReader(payload.Encode()))
defer cancel()
if err != nil {
return err
}
client := &http.Client{}
req.Header.Add("X-ChatWorkToken", token)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
resp, err := client.Do(reqs)
client, err := util.GetHTTPClient(config.Conf.HTTPProxy)
if err != nil {
return err
}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}

4
report/cve_client.go
View File

@@ -114,7 +114,7 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh
var resp *http.Response
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Get(url).End()
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return xerrors.Errorf("HTTP GET Error, url: %s, resp: %v, err: %s",
url, resp, errs)
@@ -162,7 +162,7 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c
var resp *http.Response
f := func() (err error) {
// req := gorequest.New().SetDebug(config.Conf.Debug).Post(url)
req := gorequest.New().Post(url)
req := gorequest.New().Timeout(10 * time.Second).Post(url)
for key := range query {
req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json")
}

2
report/slack.go
View File

@@ -136,7 +136,7 @@ func send(msg message) error {
jsonBody := string(bytes)
f := func() (err error) {
resp, body, errs := gorequest.New().Proxy(config.Conf.HTTPProxy).Post(conf.HookURL).Send(string(jsonBody)).End()
resp, body, errs := gorequest.New().Timeout(10 * time.Second).Proxy(config.Conf.HTTPProxy).Post(conf.HookURL).Send(string(jsonBody)).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {

17
report/telegram.go
View File

@@ -2,13 +2,16 @@ package report
import (
"bytes"
"context"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
"golang.org/x/xerrors"
)
@@ -55,15 +58,21 @@ func (w TelegramWriter) Write(rs ...models.ScanResult) (err error) {
func sendMessage(chatID, token, message string) error {
uri := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", token)
payload := `{"text": "` + strings.Replace(message, `"`, `\"`, -1) + `", "chat_id": "` + chatID + `", "parse_mode": "Markdown" }`
req, err := http.NewRequest("POST", uri, bytes.NewBuffer([]byte(payload)))
req.Header.Add("Content-Type", "application/json")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, bytes.NewBuffer([]byte(payload)))
defer cancel()
if err != nil {
return err
}
req.Header.Add("Content-Type", "application/json")
client, err := util.GetHTTPClient(config.Conf.HTTPProxy)
if err != nil {
return err
}
client := &http.Client{}
resp, err := client.Do(req)
if checkResponse(resp) != nil && err != nil {
fmt.Println(err)
return err
}
defer resp.Body.Close()

70
saas/saas.go
View File

@@ -2,20 +2,22 @@ package saas
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/future-architect/vuls/config"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
@@ -41,8 +43,7 @@ type payload struct {
}
// UploadSaas : UploadSaas
func (w Writer) Write(rs ...models.ScanResult) (err error) {
// dir string, configPath string, config *c.Config
func (w Writer) Write(rs ...models.ScanResult) error {
if len(rs) == 0 {
return nil
}
@@ -60,34 +61,25 @@ func (w Writer) Write(rs ...models.ScanResult) (err error) {
ScannedIPv4s: strings.Join(ipv4s, ", "),
ScannedIPv6s: strings.Join(ipv6s, ", "),
}
var body []byte
if body, err = json.Marshal(payload); err != nil {
body, err := json.Marshal(payload)
if err != nil {
return xerrors.Errorf("Failed to Marshal to JSON: %w", err)
}
var req *http.Request
if req, err = http.NewRequest("POST", c.Conf.Saas.URL, bytes.NewBuffer(body)); err != nil {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.Conf.Saas.URL, bytes.NewBuffer(body))
defer cancel()
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
proxy := c.Conf.HTTPProxy
var client http.Client
if proxy != "" {
proxyURL, _ := url.Parse(proxy)
client = http.Client{
Transport: &http.Transport{
Proxy: http.ProxyURL(proxyURL),
},
}
} else {
client = http.Client{}
client, err := util.GetHTTPClient(config.Conf.HTTPProxy)
if err != nil {
return err
}
var resp *http.Response
if resp, err = client.Do(req); err != nil {
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
@@ -95,44 +87,40 @@ func (w Writer) Write(rs ...models.ScanResult) (err error) {
return xerrors.Errorf("Failed to get Credential. Request JSON : %s,", string(body))
}
var t []byte
if t, err = ioutil.ReadAll(resp.Body); err != nil {
t, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
var tempCredential TempCredential
if err = json.Unmarshal(t, &tempCredential); err != nil {
if err := json.Unmarshal(t, &tempCredential); err != nil {
return xerrors.Errorf("Failed to unmarshal saas credential file. err : %s", err)
}
credential := credentials.NewStaticCredentialsFromCreds(credentials.Value{
AccessKeyID: *tempCredential.Credential.AccessKeyId,
SecretAccessKey: *tempCredential.Credential.SecretAccessKey,
SessionToken: *tempCredential.Credential.SessionToken,
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentialsFromCreds(credentials.Value{
AccessKeyID: *tempCredential.Credential.AccessKeyId,
SecretAccessKey: *tempCredential.Credential.SecretAccessKey,
SessionToken: *tempCredential.Credential.SessionToken,
}),
Region: aws.String("ap-northeast-1"),
})
var sess *session.Session
if sess, err = session.NewSession(&aws.Config{
Credentials: credential,
Region: aws.String("ap-northeast-1"),
}); err != nil {
if err != nil {
return xerrors.Errorf("Failed to new aws session. err: %w", err)
}
svc := s3.New(sess)
for _, r := range rs {
s3Key := renameKeyName(r.ServerUUID, r.Container)
var b []byte
if b, err = json.Marshal(r); err != nil {
b, err := json.Marshal(r)
if err != nil {
return xerrors.Errorf("Failed to Marshal to JSON: %w", err)
}
util.Log.Infof("Uploading...: ServerName: %s, ", r.ServerName)
s3Key := renameKeyName(r.ServerUUID, r.Container)
putObjectInput := &s3.PutObjectInput{
Bucket: aws.String(tempCredential.S3Bucket),
Key: aws.String(path.Join(tempCredential.S3ResultsDir, s3Key)),
Body: bytes.NewReader(b),
}
if _, err := svc.PutObject(putObjectInput); err != nil {
return xerrors.Errorf("Failed to upload data to %s/%s, err: %w",
tempCredential.S3Bucket, s3Key, err)

17
util/util.go
View File

@@ -3,6 +3,7 @@ package util
import (
"fmt"
"net"
"net/http"
"net/url"
"strings"
@@ -178,3 +179,19 @@ func Major(version string) string {
}
return ver[0:strings.Index(ver, ".")]
}
// GetHttpClient return http.Client
func GetHTTPClient(proxy string) (*http.Client, error) {
if proxy != "" {
proxyURL, err := url.Parse(proxy)
if err != nil {
return nil, err
}
return &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyURL(proxyURL),
},
}, nil
}
return &http.Client{}, nil
}

13
wordpress/wordpress.go
View File

@@ -1,12 +1,15 @@
package wordpress
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"time"
"github.com/future-architect/vuls/config"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/errof"
"github.com/future-architect/vuls/models"
@@ -216,12 +219,18 @@ func extractToVulnInfos(pkgName string, cves []WpCveInfo) (vinfos []models.VulnI
}
func httpRequest(url, token string) (string, error) {
req, err := http.NewRequest("GET", url, nil)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
defer cancel()
if err != nil {
return "", err
}
req.Header.Set("Authorization", fmt.Sprintf("Token token=%s", token))
resp, err := new(http.Client).Do(req)
client, err := util.GetHTTPClient(config.Conf.HTTPProxy)
if err != nil {
return "", err
}
resp, err := client.Do(req)
if err != nil {
return "", err
}