Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Continue scanning even when some hosts have tech issues

Browse Source 
see #264
This commit is contained in:
Kota Kanbe
2017-01-31 14:35:16 +09:00
parent 00660485b7
commit 386b97d2be
19 changed files with 447 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

264
scan/serverapi.go
View File

@@ -25,17 +25,14 @@ import (
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/future-architect/vuls/cache"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/report"
"github.com/future-architect/vuls/util"
)
// Log for localhsot
var Log *logrus.Entry
var servers []osTypeInterface
var servers, errServers []osTypeInterface
// Base Interface of redhat, debian, freebsd
type osTypeInterface interface {
@@ -50,13 +47,13 @@ type osTypeInterface interface {
getLackDependencies() []string
checkIfSudoNoPasswd() error
detectPlatform() error
detectPlatform()
getPlatform() models.Platform
checkRequiredPackagesInstalled() error
scanPackages() error
install() error
convertToModel() (models.ScanResult, error)
convertToModel() models.ScanResult
runningContainers() ([]config.Container, error)
exitedContainers() ([]config.Container, error)
@@ -89,33 +86,32 @@ func detectOS(c config.ServerInfo) (osType osTypeInterface) {
itsMe, osType, fatalErr = detectDebian(c)
if fatalErr != nil {
osType.setServerInfo(c)
osType.setErrs([]error{fatalErr})
return
} else if itsMe {
Log.Debugf("Debian like Linux. Host: %s:%s", c.Host, c.Port)
}
if itsMe {
util.Log.Debugf("Debian like Linux. Host: %s:%s", c.Host, c.Port)
return
}
if itsMe, osType = detectRedhat(c); itsMe {
Log.Debugf("Redhat like Linux. Host: %s:%s", c.Host, c.Port)
util.Log.Debugf("Redhat like Linux. Host: %s:%s", c.Host, c.Port)
return
}
if itsMe, osType = detectFreebsd(c); itsMe {
Log.Debugf("FreeBSD. Host: %s:%s", c.Host, c.Port)
util.Log.Debugf("FreeBSD. Host: %s:%s", c.Host, c.Port)
return
}
//TODO darwin https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/darwin.rb
osType.setServerInfo(c)
osType.setErrs([]error{fmt.Errorf("Unknown OS Type")})
return
}
// PrintSSHableServerNames print SSH-able servernames
func PrintSSHableServerNames() {
Log.Info("SSH-able servers are below...")
util.Log.Info("SSH-able servers are below...")
for _, s := range servers {
if s.getServerInfo().IsContainer() {
fmt.Printf("%s@%s ",
@@ -130,67 +126,74 @@ func PrintSSHableServerNames() {
}
// InitServers detect the kind of OS distribution of target servers
func InitServers(localLogger *logrus.Entry) error {
Log = localLogger
servers = detectServerOSes()
func InitServers() error {
servers, errServers = detectServerOSes()
if len(servers) == 0 {
return fmt.Errorf("No scannable servers")
}
containers := detectContainerOSes()
actives, inactives := detectContainerOSes()
if config.Conf.ContainersOnly {
servers = containers
servers = actives
errServers = inactives
} else {
servers = append(servers, containers...)
servers = append(servers, actives...)
errServers = append(errServers, inactives...)
}
return nil
}
func detectServerOSes() (sshAbleOses []osTypeInterface) {
Log.Info("Detecting OS of servers... ")
func detectServerOSes() (servers, errServers []osTypeInterface) {
util.Log.Info("Detecting OS of servers... ")
osTypeChan := make(chan osTypeInterface, len(config.Conf.Servers))
defer close(osTypeChan)
for _, s := range config.Conf.Servers {
go func(s config.ServerInfo) {
defer func() {
if p := recover(); p != nil {
Log.Debugf("Panic: %s on %s", p, s.ServerName)
util.Log.Debugf("Panic: %s on %s", p, s.ServerName)
}
}()
osTypeChan <- detectOS(s)
}(s)
}
var oses []osTypeInterface
timeout := time.After(30 * time.Second)
for i := 0; i < len(config.Conf.Servers); i++ {
select {
case res := <-osTypeChan:
oses = append(oses, res)
if 0 < len(res.getErrs()) {
Log.Errorf("(%d/%d) Failed: %s, err: %s",
errServers = append(errServers, res)
util.Log.Errorf("(%d/%d) Failed: %s, err: %s",
i+1, len(config.Conf.Servers),
res.getServerInfo().ServerName,
res.getErrs())
} else {
Log.Infof("(%d/%d) Detected: %s: %s",
servers = append(servers, res)
util.Log.Infof("(%d/%d) Detected: %s: %s",
i+1, len(config.Conf.Servers),
res.getServerInfo().ServerName,
res.getDistro())
}
case <-timeout:
msg := "Timed out while detecting servers"
Log.Error(msg)
for servername := range config.Conf.Servers {
util.Log.Error(msg)
for servername, sInfo := range config.Conf.Servers {
found := false
for _, o := range oses {
for _, o := range append(servers, errServers...) {
if servername == o.getServerInfo().ServerName {
found = true
break
}
}
if !found {
Log.Errorf("(%d/%d) Timed out: %s",
u := &unknown{}
u.setServerInfo(sInfo)
u.setErrs([]error{
fmt.Errorf("Timed out"),
})
errServers = append(errServers, u)
util.Log.Errorf("(%d/%d) Timed out: %s",
i+1, len(config.Conf.Servers),
servername)
i++
@@ -198,24 +201,18 @@ func detectServerOSes() (sshAbleOses []osTypeInterface) {
}
}
}
for _, o := range oses {
if len(o.getErrs()) == 0 {
sshAbleOses = append(sshAbleOses, o)
}
}
return
}
func detectContainerOSes() (actives []osTypeInterface) {
Log.Info("Detecting OS of containers... ")
func detectContainerOSes() (actives, inactives []osTypeInterface) {
util.Log.Info("Detecting OS of containers... ")
osTypesChan := make(chan []osTypeInterface, len(servers))
defer close(osTypesChan)
for _, s := range servers {
go func(s osTypeInterface) {
defer func() {
if p := recover(); p != nil {
Log.Debugf("Panic: %s on %s",
util.Log.Debugf("Panic: %s on %s",
p, s.getServerInfo().GetServerName())
}
}()
@@ -223,7 +220,6 @@ func detectContainerOSes() (actives []osTypeInterface) {
}(s)
}
var oses []osTypeInterface
timeout := time.After(30 * time.Second)
for i := 0; i < len(servers); i++ {
select {
@@ -231,36 +227,38 @@ func detectContainerOSes() (actives []osTypeInterface) {
for _, osi := range res {
sinfo := osi.getServerInfo()
if 0 < len(osi.getErrs()) {
Log.Errorf("Failed: %s err: %s", sinfo.ServerName, osi.getErrs())
inactives = append(inactives, osi)
util.Log.Errorf("Failed: %s err: %s", sinfo.ServerName, osi.getErrs())
continue
}
oses = append(oses, osi)
Log.Infof("Detected: %s@%s: %s",
actives = append(actives, osi)
util.Log.Infof("Detected: %s@%s: %s",
sinfo.Container.Name, sinfo.ServerName, osi.getDistro())
}
case <-timeout:
msg := "Timed out while detecting containers"
Log.Error(msg)
for servername := range config.Conf.Servers {
util.Log.Error(msg)
for servername, sInfo := range config.Conf.Servers {
found := false
for _, o := range oses {
for _, o := range append(actives, inactives...) {
if servername == o.getServerInfo().ServerName {
found = true
break
}
}
if !found {
Log.Errorf("Timed out: %s", servername)
u := &unknown{}
u.setServerInfo(sInfo)
u.setErrs([]error{
fmt.Errorf("Timed out"),
})
inactives = append(inactives)
util.Log.Errorf("Timed out: %s", servername)
}
}
}
}
for _, o := range oses {
if len(o.getErrs()) == 0 {
actives = append(actives, o)
}
}
return
}
@@ -339,28 +337,20 @@ func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeIn
}
// CheckIfSudoNoPasswd checks whether vuls can sudo with nopassword via SSH
func CheckIfSudoNoPasswd(localLogger *logrus.Entry) error {
func CheckIfSudoNoPasswd() {
timeoutSec := 15
errs := parallelSSHExec(func(o osTypeInterface) error {
parallelExec(func(o osTypeInterface) error {
return o.checkIfSudoNoPasswd()
}, timeoutSec)
if 0 < len(errs) {
return fmt.Errorf(fmt.Sprintf("%s", errs))
}
return nil
return
}
// DetectPlatforms detects the platform of each servers.
func DetectPlatforms(localLogger *logrus.Entry) {
errs := detectPlatforms()
if 0 < len(errs) {
// Only logging
Log.Warnf("Failed to detect platforms. err: %v", errs)
}
func DetectPlatforms() {
detectPlatforms()
for i, s := range servers {
if s.getServerInfo().IsContainer() {
Log.Infof("(%d/%d) %s on %s is running on %s",
util.Log.Infof("(%d/%d) %s on %s is running on %s",
i+1, len(servers),
s.getServerInfo().Container.Name,
s.getServerInfo().ServerName,
@@ -368,7 +358,7 @@ func DetectPlatforms(localLogger *logrus.Entry) {
)
} else {
Log.Infof("(%d/%d) %s is running on %s",
util.Log.Infof("(%d/%d) %s is running on %s",
i+1, len(servers),
s.getServerInfo().ServerName,
s.getPlatform().Name,
@@ -378,52 +368,68 @@ func DetectPlatforms(localLogger *logrus.Entry) {
return
}
func detectPlatforms() []error {
func detectPlatforms() {
timeoutSec := 1 * 60
return parallelSSHExec(func(o osTypeInterface) error {
return o.detectPlatform()
parallelExec(func(o osTypeInterface) error {
o.detectPlatform()
// Logging only if platform can not be specified
return nil
}, timeoutSec)
return
}
// Prepare installs requred packages to scan vulnerabilities.
func Prepare() []error {
errs := parallelSSHExec(func(o osTypeInterface) error {
func Prepare() error {
parallelExec(func(o osTypeInterface) error {
if err := o.checkDependencies(); err != nil {
return err
}
return nil
})
if len(errs) != 0 {
return errs
}
var targets []osTypeInterface
var targets, nonTargets []osTypeInterface
for _, s := range servers {
deps := s.getLackDependencies()
if len(deps) != 0 {
targets = append(targets, s)
} else {
nonTargets = append(nonTargets, s)
}
}
if len(targets) == 0 {
Log.Info("No need to install dependencies")
if 0 < len(nonTargets) {
util.Log.Info("The following servers were already installed dependencies")
for _, s := range nonTargets {
util.Log.Infof(" - %s", s.getServerInfo().GetServerName())
}
}
if 0 < len(errServers) {
util.Log.Error("Some errors occurred in the following servers")
for _, s := range errServers {
util.Log.Errorf(" - %s", s.getServerInfo().GetServerName())
}
} else {
util.Log.Info("Success")
}
return nil
}
Log.Info("The following servers need dependencies installed")
util.Log.Info("The following servers need to install dependencies")
for _, s := range targets {
for _, d := range s.getLackDependencies() {
Log.Infof(" - %s on %s", d, s.getServerInfo().GetServerName())
util.Log.Infof(" - %s on %s", d, s.getServerInfo().GetServerName())
}
}
if !config.Conf.AssumeYes {
Log.Info("Is this ok to install dependencies on the servers? [y/N]")
util.Log.Info("Is this ok to install dependencies on the servers? [y/N]")
reader := bufio.NewReader(os.Stdin)
for {
text, err := reader.ReadString('\n')
if err != nil {
return []error{err}
return err
}
switch strings.TrimSpace(text) {
case "", "N", "n":
@@ -431,62 +437,81 @@ func Prepare() []error {
case "y", "Y":
goto yes
default:
Log.Info("Please enter y or N")
util.Log.Info("Please enter y or N")
}
}
}
yes:
servers = targets
errs = parallelSSHExec(func(o osTypeInterface) error {
parallelExec(func(o osTypeInterface) error {
if err := o.install(); err != nil {
return err
}
return nil
})
if len(errs) != 0 {
return errs
if 0 < len(servers) {
util.Log.Info("Successfully installed in the followring servers")
for _, s := range servers {
util.Log.Infof(" - %s", s.getServerInfo().GetServerName())
}
}
if 0 < len(nonTargets) {
util.Log.Info("The following servers were already installed dependencies")
for _, s := range nonTargets {
util.Log.Infof(" - %s", s.getServerInfo().GetServerName())
}
}
if 0 < len(errServers) {
util.Log.Error("Some errors occurred in the following servers")
for _, s := range errServers {
util.Log.Errorf(" - %s", s.getServerInfo().GetServerName())
}
}
if len(errServers) == 0 {
util.Log.Info("Success")
} else {
util.Log.Error("Failure")
}
Log.Info("All dependencies were installed correctly")
return nil
}
// Scan scan
func Scan() []error {
func Scan() error {
if len(servers) == 0 {
return []error{fmt.Errorf("No server defined. Check the configuration")}
return fmt.Errorf("No server defined. Check the configuration")
}
Log.Info("Check required packages for scanning...")
if errs := checkRequiredPackagesInstalled(); errs != nil {
Log.Error("Please execute with [prepare] subcommand to install required packages before scanning")
return errs
}
util.Log.Info("Check required packages for scanning...")
checkRequiredPackagesInstalled()
if err := setupCangelogCache(); err != nil {
return []error{err}
if err := setupChangelogCache(); err != nil {
return err
}
defer func() {
if cache.DB != nil {
cache.DB.Close()
}
}()
Log.Info("Scanning vulnerable OS packages...")
util.Log.Info("Scanning vulnerable OS packages...")
scannedAt := time.Now()
dir, err := ensureResultDir(scannedAt)
if err != nil {
return []error{err}
return err
}
if errs := scanVulns(dir, scannedAt); errs != nil {
return errs
if err := scanVulns(dir, scannedAt); err != nil {
return err
}
return nil
}
func setupCangelogCache() error {
func setupChangelogCache() error {
needToSetupCache := false
for _, s := range servers {
switch s.getDistro().Family {
@@ -496,7 +521,7 @@ func setupCangelogCache() error {
}
}
if needToSetupCache {
if err := cache.SetupBolt(config.Conf.CacheDBPath, Log); err != nil {
if err := cache.SetupBolt(config.Conf.CacheDBPath, util.Log); err != nil {
return err
}
}
@@ -505,28 +530,24 @@ func setupCangelogCache() error {
func checkRequiredPackagesInstalled() []error {
timeoutSec := 30 * 60
return parallelSSHExec(func(o osTypeInterface) error {
parallelExec(func(o osTypeInterface) error {
return o.checkRequiredPackagesInstalled()
}, timeoutSec)
return nil
}
func scanVulns(jsonDir string, scannedAt time.Time) []error {
func scanVulns(jsonDir string, scannedAt time.Time) error {
var results models.ScanResults
timeoutSec := 120 * 60
errs := parallelSSHExec(func(o osTypeInterface) error {
if err := o.scanPackages(); err != nil {
return err
}
parallelExec(func(o osTypeInterface) error {
return o.scanPackages()
}, timeoutSec)
r, err := o.convertToModel()
if err != nil {
return err
}
for _, s := range append(servers, errServers...) {
r := s.convertToModel()
r.ScannedAt = scannedAt
results = append(results, r)
return nil
}, timeoutSec)
}
config.Conf.FormatJSON = true
ws := []report.ResultWriter{
@@ -534,14 +555,9 @@ func scanVulns(jsonDir string, scannedAt time.Time) []error {
}
for _, w := range ws {
if err := w.Write(results...); err != nil {
return []error{
fmt.Errorf("Failed to write summary report: %s", err),
}
return fmt.Errorf("Failed to write summary report: %s", err)
}
}
if errs != nil {
return errs
}
report.StdoutWriter{}.WriteScanSummary(results...)
return nil