diff --git a/.travis.yml b/.travis.yml index c4552c6f..2875f639 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.7 - 1.8 diff --git a/GNUmakefile b/GNUmakefile index 3cc9b7ae..d44cb749 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -15,7 +15,7 @@ clean SRCS = $(shell git ls-files '*.go') -PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands ./cache +PKGS = ./. ./cache ./commands ./config ./models ./oval ./report ./scan ./util VERSION := $(shell git describe --tags --abbrev=0) REVISION := $(shell git rev-parse --short HEAD) LDFLAGS := -X 'main.version=$(VERSION)' \ diff --git a/Gopkg.lock b/Gopkg.lock index 73acf56d..4829d8f7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,16 +1,17 @@ -memo = "404d058cf6b46d820e153afc5721e1ab2aa2b10ec345d969b9c460097f99add3" +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + [[projects]] - branch = "master" - name = "github.com/Azure/azure-storage-go" - packages = ["."] - revision = "12ccaadb081cdd217702067d28da9a7ff4305239" + name = "github.com/Azure/azure-sdk-for-go" + packages = ["storage"] + revision = "57db66900881e9fd21fd041a9d013514700ecab3" + version = "v10.3.0-beta" [[projects]] name = "github.com/Azure/go-autorest" - packages = ["autorest","autorest/azure","autorest/date"] - revision = "a2fdd780c9a50455cecd249b00bdc3eb73a78e31" - version = "v7.3.1" + packages = ["autorest","autorest/adal","autorest/azure","autorest/date"] + revision = "77a52603f06947221c672f10275abc9bf2c7d557" + version = "v8.3.0" [[projects]] name = "github.com/BurntSushi/toml" @@ -18,40 +19,35 @@ memo = "404d058cf6b46d820e153afc5721e1ab2aa2b10ec345d969b9c460097f99add3" revision = "b26d9c308763d68093482582cea63d69be07a0f0" version = "v0.3.0" -[[projects]] - branch = "master" - name = "github.com/Sirupsen/logrus" - packages = ["."] - revision = "10f801ebc38b33738c9d17d50860f484a0988ff5" - [[projects]] name = "github.com/asaskevich/govalidator" packages = ["."] - revision = "7b3beb6df3c42abd3509abfc3bcacc0fbfb7c877" - version = "v5" + revision = "4918b99a7cb949bb295f3c7bbaf24b577d806e35" + version = "v6" [[projects]] name = "github.com/aws/aws-sdk-go" - packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/request","aws/session","aws/signer/v4","private/endpoints","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","private/waiter","service/s3","service/sts"] - revision = "5b341290c488aa6bd76b335d819b4a68516ec3ab" + packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"] + revision = "264af29009637e0a9e5d4a276d0969c3ed918ffd" + version = "v1.10.29" [[projects]] name = "github.com/boltdb/bolt" packages = ["."] - revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9" - version = "v1.3.0" + revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" + version = "v1.3.1" [[projects]] name = "github.com/cenkalti/backoff" packages = ["."] - revision = "32cd0c5b3aef12c76ed64aaf678f6c79736be7dc" - version = "v1.0.0" + revision = "61153c768f31ee5f130071d08fc82b85208528de" + version = "v1.1.0" [[projects]] name = "github.com/cheggaaa/pb" packages = ["."] - revision = "b6229822fa186496fcbf34111237e7a9693c6971" - version = "v1.0.13" + revision = "0d6285554e726cc0620cbecc7e6969c945dcc63b" + version = "v1.0.17" [[projects]] name = "github.com/dgrijalva/jwt-go" @@ -62,8 +58,14 @@ memo = "404d058cf6b46d820e153afc5721e1ab2aa2b10ec345d969b9c460097f99add3" [[projects]] name = "github.com/go-ini/ini" packages = ["."] - revision = "e7fea39b01aea8d5671f6858f0532f56e8bff3a5" - version = "v1.27.0" + revision = "20b96f641a5ea98f2f8619ff4f3e061cff4833bd" + version = "v1.28.2" + +[[projects]] + name = "github.com/go-redis/redis" + packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto"] + revision = "19c1c2272e00c1aaa903cf574c746cd449f9cd3c" + version = "v6.5.7" [[projects]] name = "github.com/go-sql-driver/mysql" @@ -108,22 +110,34 @@ memo = "404d058cf6b46d820e153afc5721e1ab2aa2b10ec345d969b9c460097f99add3" version = "0.2.2" [[projects]] - branch = "master" name = "github.com/jroimartin/gocui" packages = ["."] - revision = "612b0b2987ec1a6af46d7008cef1efd4b3898346" + revision = "4e9ce9a8e26f2ef33dfe297dbdfca148733b6b9b" + version = "v0.3.0" [[projects]] + branch = "master" name = "github.com/k0kubun/pp" packages = ["."] - revision = "027a6d1765d673d337e687394dbe780dd64e2a1e" - version = "v2.3.0" + revision = "d1532fc5d94ecdf2da29e24d7b99721f3287de4a" + +[[projects]] + branch = "master" + name = "github.com/knqyf263/go-deb-version" + packages = ["."] + revision = "9865fe14d09b1c729188ac810466dde90f897ee3" + +[[projects]] + branch = "master" + name = "github.com/knqyf263/go-rpm-version" + packages = ["."] + revision = "74609b86c936dff800c69ec89fcf4bc52d5f13a4" [[projects]] branch = "master" name = "github.com/kotakanbe/go-cve-dictionary" packages = ["config","db","jvn","log","models","nvd","util"] - revision = "d47709be4cc24d2c77a7be9096dcfcf211ba1d57" + revision = "c20fa7e1d07f7c700baf12c855f7fcf61525f1b6" [[projects]] name = "github.com/kotakanbe/go-pingscanner" @@ -131,23 +145,35 @@ memo = "404d058cf6b46d820e153afc5721e1ab2aa2b10ec345d969b9c460097f99add3" revision = "641dc2cc2d3cbf295dad356667b74c69bcbd6f70" version = "v0.1.0" +[[projects]] + branch = "master" + name = "github.com/kotakanbe/goval-dictionary" + packages = ["config","db","db/rdb","log","models"] + revision = "3523cc174e68f285d0572d07c68ffa3a9290799c" + [[projects]] branch = "master" name = "github.com/kotakanbe/logrus-prefixed-formatter" packages = ["."] - revision = "e7519b8c80ba008a3bfc57ffa31232bf2a77f455" + revision = "75edb2e85a38873f0318be05a458446681d1022f" + +[[projects]] + name = "github.com/labstack/gommon" + packages = ["color","log"] + revision = "779b8a8b9850a97acba6a3fe20feb628c39e17c1" + version = "0.2.2" [[projects]] branch = "master" name = "github.com/lib/pq" packages = [".","hstore","oid"] - revision = "2704adc878c21e1329f46f6e56a1c387d788ff94" + revision = "e42267488fe361b9dc034be7a6bffef5b195bceb" [[projects]] name = "github.com/mattn/go-colorable" packages = ["."] - revision = "d228849504861217f796da67fae4f6e347643f15" - version = "v0.0.7" + revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" + version = "v0.0.9" [[projects]] name = "github.com/mattn/go-isatty" @@ -183,7 +209,7 @@ memo = "404d058cf6b46d820e153afc5721e1ab2aa2b10ec345d969b9c460097f99add3" branch = "master" name = "github.com/nsf/termbox-go" packages = ["."] - revision = "7994c181db7761ca3c67a217068cf31826113f5f" + revision = "4ed959e0540971545eddb8c75514973d670cf739" [[projects]] name = "github.com/parnurzeal/gorequest" @@ -200,29 +226,66 @@ memo = "404d058cf6b46d820e153afc5721e1ab2aa2b10ec345d969b9c460097f99add3" [[projects]] name = "github.com/rifflock/lfshook" packages = ["."] - revision = "2adb3e0c4ddd8778c4adde609d2dfd4fbe6096ea" - version = "1.6" + revision = "6844c808343cb8fa357d7f141b1b990e05d24e41" + version = "1.7" + +[[projects]] + name = "github.com/satori/uuid" + packages = ["."] + revision = "879c5887cd475cd7864858769793b2ceb0d44feb" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/sirupsen/logrus" + packages = ["."] + revision = "84573d5f03ab3740f524c7842c3a9bf617961d32" + +[[projects]] + branch = "master" + name = "github.com/valyala/bytebufferpool" + packages = ["."] + revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" + +[[projects]] + branch = "master" + name = "github.com/valyala/fasttemplate" + packages = ["."] + revision = "dcecefd839c4193db0d35b88ec65b4c12d360ab0" + +[[projects]] + branch = "master" + name = "github.com/ymomoi/goval-parser" + packages = ["oval"] + revision = "0a0be1dd9d0855b50be0be5a10ad3085382b6d59" [[projects]] branch = "master" name = "golang.org/x/crypto" packages = ["curve25519","ed25519","ed25519/internal/edwards25519","ssh","ssh/agent","ssh/terminal"] - revision = "ed779e1bec0180cdfce8135ca6558067b388777b" + revision = "eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035" [[projects]] branch = "master" name = "golang.org/x/net" packages = ["context","idna","publicsuffix"] - revision = "d1e1b351919c6738fdeb9893d5c998b161464f0c" + revision = "1c05540f6879653db88113bc4a2b70aec4bd491f" [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix"] - revision = "f3918c30c5c2cb527c0b071a27c35120a6c0719a" + packages = ["unix","windows"] + revision = "07c182904dbd53199946ba614a412c61d3c548f5" [[projects]] branch = "master" name = "golang.org/x/text" packages = ["internal/gen","internal/triegen","internal/ucd","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] - revision = "f4b4367115ec2de254587813edaa901bc1c723a8" + revision = "e56139fd9c5bc7244c76116c68e500765bb6db6b" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "36d700add80d36c56484ed310b9a7e622b3e308ab22eb42bdfb02fd8f5c90407" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index b1a1ac21..7f770516 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,28 +1,90 @@ -[[dependencies]] - branch = "master" - name = "github.com/Azure/azure-storage-go" +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" -[[dependencies]] - branch = "master" + +[[constraint]] name = "github.com/BurntSushi/toml" + version = "0.3.0" -[[dependencies]] +[[constraint]] + name = "github.com/asaskevich/govalidator" + version = "6.0.0" + +[[constraint]] + name = "github.com/boltdb/bolt" + version = "1.3.1" + +[[constraint]] + name = "github.com/cenkalti/backoff" + version = "1.0.0" + +[[constraint]] branch = "master" - name = "github.com/Sirupsen/logrus" + name = "github.com/google/subcommands" -[[dependencies]] - name = "github.com/aws/aws-sdk-go" - revision = "5b341290c488aa6bd76b335d819b4a68516ec3ab" - -[[dependencies]] +[[constraint]] branch = "master" + name = "github.com/gosuri/uitable" + +[[constraint]] + branch = "master" + name = "github.com/howeyc/gopass" + +[[constraint]] name = "github.com/jroimartin/gocui" + version = "0.3.0" -[[dependencies]] +[[constraint]] branch = "master" - name = "github.com/kotakanbe/go-cve-dictionary" + name = "github.com/k0kubun/pp" -[[dependencies]] +[[constraint]] + branch = "master" + name = "github.com/knqyf263/go-deb-version" + +[[constraint]] + branch = "master" + name = "github.com/knqyf263/go-rpm-version" + +[[constraint]] + name = "github.com/kotakanbe/go-pingscanner" + version = "0.1.0" + +[[constraint]] branch = "master" name = "github.com/kotakanbe/logrus-prefixed-formatter" + +[[constraint]] + name = "github.com/parnurzeal/gorequest" + version = "0.2.15" + +[[constraint]] + name = "github.com/rifflock/lfshook" + version = "1.7.0" + +[[constraint]] + branch = "master" + name = "github.com/sirupsen/logrus" + +[[constraint]] + branch = "master" + name = "github.com/kotakanbe/go-cve-dictionary" diff --git a/README.ja.md b/README.ja.md index 0e9c3473..81d7178e 100644 --- a/README.ja.md +++ b/README.ja.md @@ -8,7 +8,10 @@ Vulnerability scanner for Linux/FreeBSD, agentless, written in golang. [README in English](https://github.com/future-architect/vuls/blob/master/README.md) -Slackチームは[こちらから](http://goo.gl/forms/xm5KFo35tu)参加できます。(日本語でオッケーです) +Slackチームは[こちらから](http://goo.gl/forms/xm5KFo35tu)参加できます。(日本語でオッケーです) +Twitter: 日本語: [@vuls_ja](https://twitter.com/vuls_ja), 英語: [@vuls_en](https://twitter.com/vuls_en) + +![Vuls-Abstract](img/vuls-abstract.png) [![asciicast](https://asciinema.org/a/bazozlxrw1wtxfu9yojyihick.png)](https://asciinema.org/a/bazozlxrw1wtxfu9yojyihick) @@ -18,89 +21,98 @@ Slackチームは[こちらから](http://goo.gl/forms/xm5KFo35tu)参加でき # TOC -- [Vuls: VULnerability Scanner](#vuls-vulnerability-scanner) -- [TOC](#toc) -- [Abstract](#abstract) -- [Main Features](#main-features) -- [What Vuls Doesn't Do](#what-vuls-doesnt-do) -- [Setup Vuls](#setup-vuls) -- [Tutorial: Local Scan Mode](#tutorial-local-scan-mode) - * [Step1. Launch Amazon Linux](#step1-launch-amazon-linux) - * [Step2. Install requirements](#step2-install-requirements) - * [Step3. Deploy go-cve-dictionary](#step3-deploy-go-cve-dictionary) - * [Step4. Deploy Vuls](#step4-deploy-vuls) - * [Step5. Config](#step5-config) - * [Step6. Check config.toml and settings on the server before scanning](#step6-check-configtoml-and-settings-on-the-server-before-scanning) - * [Step7. Start Scanning](#step7-start-scanning) - * [Step8. Reporting](#step8-reporting) - * [Step9. TUI](#step9-tui) - * [Step10. Web UI](#step10-web-ui) -- [Tutorial: Remote Scan Mode](#tutorial-remote-scan-mode) - * [Step1. Launch Another Amazon Linux](#step1-launch-another-amazon-linux) - * [Step2. Install Dependencies on the Remote Server](#step2-install-dependencies-on-the-remote-server) - * [Step3. Enable to SSH from Localhost](#step3-enable-to-ssh-from-localhost) - * [Step4. Config](#step4-config) - * [Step5. Check config.toml and settings on the server before scanning](#step5-check-configtoml-and-settings-on-the-server-before-scanning) - * [Step6. Start Scanning](#step6-start-scanning) - * [Step7. Reporting](#step7-reporting) -- [Architecture](#architecture) - * [A. Scan via SSH Mode (Remote Scan Mode)](#a-scan-via-ssh-mode-remote-scan-mode) - * [B. Scan without SSH (Local Scan Mode)](#b-scan-without-ssh-local-scan-mode) - * [go-cve-dictionary](#go-cve-dictionary) - * [Vuls](#vuls) -- [Performance Considerations](#performance-considerations) -- [Use Cases](#use-cases) - * [Scan all servers](#scan-all-servers) - * [Scan a single server](#scan-a-single-server) -- [Support OS](#support-os) -- [Usage: Automatic Server Discovery](#usage-automatic-server-discovery) - * [Example](#example) -- [Configuration](#configuration) -- [Usage: Configtest](#usage-configtest) - * [Dependencies on Target Servers](#dependencies-on-target-servers) - * [Check /etc/sudoers](#check-etcsudoers) -- [Usage: Scan](#usage-scan) - * [-ssh-native-insecure option](#-ssh-native-insecure-option) - * [-ask-key-password option](#-ask-key-password-option) - * [Example: Scan all servers defined in config file](#example-scan-all-servers-defined-in-config-file) - * [Example: Scan specific servers](#example-scan-specific-servers) - * [Example: Scan via shell instead of SSH.](#example-scan-via-shell-instead-of-ssh) - + [cronで動かす場合](#cron%E3%81%A7%E5%8B%95%E3%81%8B%E3%81%99%E5%A0%B4%E5%90%88) - * [Example: Scan containers (Docker/LXD)](#example-scan-containers-dockerlxd) - + [Docker](#docker) - + [LXDコンテナをスキャンする場合](#lxd%E3%82%B3%E3%83%B3%E3%83%86%E3%83%8A%E3%82%92%E3%82%B9%E3%82%AD%E3%83%A3%E3%83%B3%E3%81%99%E3%82%8B%E5%A0%B4%E5%90%88) -- [Usage: Report](#usage-report) - * [How to read a report](#how-to-read-a-report) - + [Example](#example-1) - + [Summary part](#summary-part) - + [Detailed Part](#detailed-part) - + [Changelog Part](#changelog-part) - * [Example: Send scan results to Slack](#example-send-scan-results-to-slack) - * [Example: Put results in S3 bucket](#example-put-results-in-s3-bucket) - * [Example: Put results in Azure Blob storage](#example-put-results-in-azure-blob-storage) - * [Example: IgnoreCves](#example-ignorecves) - * [Example: Add optional key-value pairs to JSON](#example-add-optional-key-value-pairs-to-json) - * [Example: Use MySQL as a DB storage back-end](#example-use-mysql-as-a-db-storage-back-end) - * [Example: Use PostgreSQL as a DB storage back-end](#example-use-postgresql-as-a-db-storage-back-end) -- [Usage: Scan vulnerability of non-OS package](#usage-scan-vulnerability-of-non-os-package) -- [Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental)](#usage-integrate-with-owasp-dependency-check-to-automatic-update-when-the-libraries-are-updated-experimental) -- [Usage: TUI](#usage-tui) - * [Display the latest scan results](#display-the-latest-scan-results) - * [Display the previous scan results](#display-the-previous-scan-results) -- [Display the previous scan results using peco](#display-the-previous-scan-results-using-peco) -- [Usage: go-cve-dictionary on different server](#usage-go-cve-dictionary-on-different-server) -- [Usage: Update NVD Data](#usage-update-nvd-data) -- [レポートの日本語化](#%E3%83%AC%E3%83%9D%E3%83%BC%E3%83%88%E3%81%AE%E6%97%A5%E6%9C%AC%E8%AA%9E%E5%8C%96) - * [fetchnvd, fetchjvnの実行順序の注意](#fetchnvd-fetchjvn%E3%81%AE%E5%AE%9F%E8%A1%8C%E9%A0%86%E5%BA%8F%E3%81%AE%E6%B3%A8%E6%84%8F) - * [スキャン実行](#%E3%82%B9%E3%82%AD%E3%83%A3%E3%83%B3%E5%AE%9F%E8%A1%8C) -- [Update Vuls With Glide](#update-vuls-with-glide) -- [Misc](#misc) -- [Related Projects](#related-projects) -- [Data Source](#data-source) -- [Authors](#authors) -- [Contribute](#contribute) -- [Change Log](#change-log) -- [License](#license) +Table of Contents +================= + + * [Vuls: VULnerability Scanner](#vuls-vulnerability-scanner) + * [TOC](#toc) + * [Abstract](#abstract) + * [Main Features](#main-features) + * [What Vuls Doesn't Do](#what-vuls-doesnt-do) + * [Setup Vuls](#setup-vuls) + * [Tutorial](#tutorial) + * [Tutorial: Local Scan Mode](#tutorial-local-scan-mode) + * [Step1. Launch CentOS7](#step1-launch-centos7) + * [Step2. Install requirements](#step2-install-requirements) + * [Step3. Deploy go-cve-dictionary](#step3-deploy-go-cve-dictionary) + * [Step4. Deploy goval-dictionary](#step4-deploy-goval-dictionary) + * [Step5. Deploy Vuls](#step5-deploy-vuls) + * [Step6. Config](#step6-config) + * [Step7. Check config.toml and settings on the server before scanning](#step7-check-configtoml-and-settings-on-the-server-before-scanning) + * [Step8. Start Scanning](#step8-start-scanning) + * [Step9. Reporting](#step9-reporting) + * [Step10. TUI](#step10-tui) + * [Step11. Web UI](#step11-web-ui) + * [Tutorial: Remote Scan Mode](#tutorial-remote-scan-mode) + * [Step1. Launch new Ubuntu Linux (the server to be sacnned)](#step1-launch-new-ubuntu-linux-the-server-to-be-sacnned) + * [Step2. Enable to SSH from localhost](#step2-enable-to-ssh-from-localhost) + * [Step3. config.tomlの設定](#step3-configtomlの設定) + * [Step4. Check config.toml and settings on the server before scanning](#step4-check-configtoml-and-settings-on-the-server-before-scanning) + * [Step5. Start Scanning](#step5-start-scanning) + * [Step6. Reporting](#step6-reporting) + * [Architecture](#architecture) + * [A. Scan via SSH Mode (Remote Scan Mode)](#a-scan-via-ssh-mode-remote-scan-mode) + * [B. Scan without SSH (Local Scan Mode)](#b-scan-without-ssh-local-scan-mode) + * [Fast Scan and Deep Scan](#fast-scan-and-deep-scan) + * [Fast Scan](#fast-scan) + * [Deep Scan](#deep-scan) + * [Use Cases](#use-cases) + * [Scan all servers](#scan-all-servers) + * [Scan a single server](#scan-a-single-server) + * [Support OS](#support-os) + * [Usage: Automatic Server Discovery](#usage-automatic-server-discovery) + * [Example](#example) + * [Configuration](#configuration) + * [Usage: Configtest](#usage-configtest) + * [Fast Scan Mode](#fast-scan-mode) + * [Deep Scan Mode](#deep-scan-mode) + * [Dependencies and /etc/sudoers on Target Servers](#dependencies-and-etcsudoers-on-target-servers) + * [Usage: Scan](#usage-scan) + * [-deep option](#-deep-option) + * [-ssh-native-insecure option](#-ssh-native-insecure-option) + * [-ask-key-password option](#-ask-key-password-option) + * [Example: Scan all servers defined in config file](#example-scan-all-servers-defined-in-config-file) + * [Example: Scan specific servers](#example-scan-specific-servers) + * [Example: Scan via shell instead of SSH.](#example-scan-via-shell-instead-of-ssh) + * [cronで動かす場合](#cronで動かす場合) + * [Example: Scan containers (Docker/LXD)](#example-scan-containers-dockerlxd) + * [Docker](#docker) + * [LXDコンテナをスキャンする場合](#lxdコンテナをスキャンする場合) + * [Usage: Report](#usage-report) + * [How to read a report](#how-to-read-a-report) + * [Example](#example-1) + * [Summary part](#summary-part) + * [Detailed Part](#detailed-part) + * [Example: Send scan results to Slack](#example-send-scan-results-to-slack) + * [Example: Put results in S3 bucket](#example-put-results-in-s3-bucket) + * [Example: Put results in Azure Blob storage](#example-put-results-in-azure-blob-storage) + * [Example: IgnoreCves](#example-ignorecves) + * [Example: Add optional key-value pairs to JSON](#example-add-optional-key-value-pairs-to-json) + * [Example: Use MySQL as a DB storage back-end](#example-use-mysql-as-a-db-storage-back-end) + * [Example: Use PostgreSQL as a DB storage back-end](#example-use-postgresql-as-a-db-storage-back-end) + * [Example: Use Redis as a DB storage back-end](#example-use-redis-as-a-db-storage-back-end) + * [Usage: Scan vulnerability of non-OS package](#usage-scan-vulnerability-of-non-os-package) + * [Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental)](#usage-integrate-with-owasp-dependency-check-to-automatic-update-when-the-libraries-are-updated-experimental) + * [Usage: TUI](#usage-tui) + * [Display the latest scan results](#display-the-latest-scan-results) + * [Display the previous scan results](#display-the-previous-scan-results) + * [Display the previous scan results using peco](#display-the-previous-scan-results-using-peco) + * [Usage: go-cve-dictionary on different server](#usage-go-cve-dictionary-on-different-server) + * [Usage: Update NVD Data](#usage-update-nvd-data) + * [Usage: goval-dictionary on different server](#usage-goval-dictionary-on-different-server) + * [Usage: Update OVAL Data](#usage-update-oval-data) + * [レポートの日本語化](#レポートの日本語化) + * [fetchnvd, fetchjvnの実行順序の注意](#fetchnvd-fetchjvnの実行順序の注意) + * [スキャン実行](#スキャン実行) + * [How to Update to the Latest Version](#how-to-update-to-the-latest-version) + * [Misc](#misc) + * [Related Projects](#related-projects) + * [Data Source](#data-source) + * [Authors](#authors) + * [Contribute](#contribute) + * [Change Log](#change-log) + * [Stargazers over time](#stargazers-over-time) + * [License](#license) ---- @@ -126,16 +138,36 @@ Vulsは上に挙げた手動運用での課題を解決するツールであり # Main Features -- Linuxサーバに存在する脆弱性をスキャン - - Ubuntu, Debian, CentOS, Amazon Linux, RHEL, Raspbianに対応 +- サーバに存在する脆弱性をスキャン + - FreeBSD, Ubuntu, Debian, CentOS, Amazon Linux, RHEL, Raspbianに対応 - クラウド、オンプレミス、Docker +- 高精度なスキャン + - Vulsは複数の脆弱性データベース、複数の検知方法を組み合わせることで高精度なスキャンを実現している + - OVAL + - RHSA/ALAS/ELSA/FreeBSD-SA + - Changelog +- FastスキャンとDeepスキャン + - Fastスキャン + - root権限必要なし + - スキャン対象サーバの負荷ほぼなし + - インターネットに接続していない環境でもスキャン可能 (RedHat, CentOS, OracleLinux, Ubuntu, Debian) + - Deepスキャン + - Changelogの差分を取得し、そこに書かれているCVE-IDを検知 + - スキャン対象サーバに負荷がかかる場合がある +- リモートスキャンとローカルスキャン + - リモートスキャン + - スキャン対象サーバにSSH接続可能なマシン1台にセットアップするだけで動作 + - ローカルスキャン + - もし中央のサーバから各サーバにSSH接続できない環境の場合はローカルスキャンモードでスキャン可能 +- **動的** スキャナ + - サーバにSSH接続してコマンドを発行可能なのでサーバの状態を取得可能 + - カーネルアップデート後再起動していない場合に警告してくれる - OSパッケージ管理対象外のミドルウェアをスキャン - プログラミング言語のライブラリやフレームワーク、ミドルウェアの脆弱性スキャン - CPEに登録されているソフトウェアが対象 -- エージェントレスアーキテクチャ - - スキャン対象サーバにSSH接続可能なマシン1台にセットアップするだけで動作 - 非破壊スキャン(SSHでコマンド発行するだけ) - AWSでの脆弱性/侵入テスト事前申請は必要なし + - 毎日スケジュール実行すれば新規に公開された脆弱性にすぐに気付くことができる - 設定ファイルのテンプレート自動生成 - CIDRを指定してサーバを自動検出、設定ファイルのテンプレートを生成 - EmailやSlackで通知可能（日本語でのレポートも可能） @@ -158,7 +190,19 @@ Vulsのセットアップは以下の2パターンがある see https://github.com/future-architect/vuls/tree/master/setup/docker - 手動でセットアップ -Hello Vulsチュートリアルでは手動でのセットアップ方法で説明する +チュートリアルでは手動でのセットアップ方法で説明する + +---- + +# Tutorial + +1. Tutorial: Local Scan Mode + - Launch CentOS on AWS + - Deploy Vuls + - Scan localhost, Reporting +1. Tutorial: Remote Scan Mode + - Launch Ubuntu Linux on AWS + - このUbuntuを先程セットアップしたVulsからスキャンする ---- @@ -167,9 +211,10 @@ Hello Vulsチュートリアルでは手動でのセットアップ方法で説 本チュートリアルでは、Amazon EC2にVulsをセットアップし、自分に存在する脆弱性をスキャンする方法を説明する。 手順は以下の通り -1. Amazon Linuxを新規作成 +1. CentOSを新規作成 1. 必要なソフトウェアをインストール 1. go-cve-dictionaryをデプロイ +1. goval-dictionaryをデプロイ 1. Vulsをデプロイ 1. 設定 1. 設定ファイルと、スキャン対象サーバの設定のチェック @@ -178,9 +223,9 @@ Hello Vulsチュートリアルでは手動でのセットアップ方法で説 1. TUI(Terminal-Based User Interface)で結果を参照する 1. Web UI([VulsRepo](https://github.com/usiusi360/vulsrepo))で結果を参照する -## Step1. Launch Amazon Linux +## Step1. Launch CentOS7 -- 今回は説明のために、脆弱性を含む古いAMIを使う (amzn-ami-hvm-2015.09.1.x86_64-gp2 - ami-383c1956) +- 今回は説明のために、脆弱性を含む古いAMIを使う - EC2作成時に自動アップデートされるとVulsスキャン結果が0件になってしまうので、cloud-initに以下を指定してEC2を作成する。 ``` @@ -194,18 +239,18 @@ Hello Vulsチュートリアルでは手動でのセットアップ方法で説 Vulsセットアップに必要な以下のソフトウェアをインストールする。 -- SQLite3 or MySQL +- SQLite3, MySQL, PostgreSQL or Redis - git - gcc - GNU Make -- go v1.7.1 or later (The latest version is recommended) +- go v1.8.3 or later (The latest version is recommended) - https://golang.org/doc/install ```bash -$ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem -$ sudo yum -y install sqlite git gcc make -$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz +$ ssh centos@52.100.100.100 -i ~/.ssh/private.pem +$ sudo yum -y install sqlite git gcc make wget +$ wget https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.8.3.linux-amd64.tar.gz $ mkdir $HOME/go ``` /etc/profile.d/goenv.sh を作成し、下記を追加する。 @@ -227,7 +272,7 @@ $ source /etc/profile.d/goenv.sh ```bash $ sudo mkdir /var/log/vuls -$ sudo chown ec2-user /var/log/vuls +$ sudo chown centos /var/log/vuls $ sudo chmod 700 /var/log/vuls $ $ mkdir -p $GOPATH/src/github.com/kotakanbe @@ -237,7 +282,7 @@ $ cd go-cve-dictionary $ make install ``` バイナリは、`$GOPATH/bin`以下に生成される - +もしもインストールプロセスが途中で止まる場合は、Out of memory errorが発生している可能性があるので、インスタンスタイプを大きくして再実行してみてください。 NVDから脆弱性データベースを取得する。 環境によって異なるが、AWS上では10分程度かかる。 @@ -250,14 +295,41 @@ $ ls -alh cve.sqlite3 -rw-r--r-- 1 ec2-user ec2-user 7.0M Mar 24 13:20 cve.sqlite3 ``` -日本語化したい場合は、JVNから脆弱性データベースを取得する。 +脆弱性レポートを日本語化したい場合は、JVNから脆弱性データベースを取得する。 ```bash $ cd $HOME $ for i in `seq 1998 $(date +"%Y")`; do go-cve-dictionary fetchjvn -years $i; done ``` -## Step4. Deploy Vuls +## Step4. Deploy goval-dictionary + +[goval-dictionary](https://github.com/kotakanbe/goval-dictionary) + +```bash +$ mkdir -p $GOPATH/src/github.com/kotakanbe +$ cd $GOPATH/src/github.com/kotakanbe +$ git clone https://github.com/kotakanbe/goval-dictionary.git +$ cd goval-dictionary +$ make install +``` +The binary was built under `$GOPATH/bin` +もしもインストールプロセスが途中で止まる場合は、Out of memory errorが発生している可能性があるので、インスタンスタイプを大きくして再実行してみてください。 + +今回はCentOSがスキャン対象なので、RedHatが公開しているOVAL情報を取り込む. [README](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-redhat) + +```bash +$ goval-dictionary fetch-redhat 7 +``` + +今回はスキャン対象がCentOS 7なので、RedHat 7のOVALを取得している。 +他の種類のOSをスキャンする場合は以下を参照し、スキャン対象用のOVALを取得しておくこと +- [RedHat, CentOS](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-redhat) +- [Debian](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-debian) +- [Ubuntu](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-ubuntu) +- [Oracle Linux](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-oracle) + +## Step5. Deploy Vuls 新規にターミナルを起動し、先ほど作成したEC2にSSH接続する。 ``` @@ -267,8 +339,10 @@ $ git clone https://github.com/future-architect/vuls.git $ cd vuls $ make install ``` +The binary was built under `$GOPATH/bin` +もしもインストールプロセスが途中で止まる場合は、Out of memory errorが発生している可能性があるので、インスタンスタイプを大きくして再実行してみてください。 -## Step5. Config +## Step6. Config Vulsの設定ファイルを作成する（TOMLフォーマット） @@ -278,104 +352,101 @@ $ cat config.toml [servers] [servers.localhost] -host = "localhost" -port = "local" +host = "localhost" +port = "local" ``` -Root権限が必要なディストリビューションもあるので、スキャン対象サーバの/etc/sudoersを変更する。 -パスワードありのsudoはセキュリティ上の理由からサポートしていないので、スキャンに必要なコマンドは、`NOPASSAWORD`として、remote host上の`etc/sudoers`に定義しておく。 -See [Usage: Configtest#Check /etc/sudoers](#check-etcsudoers) - -## Step6. Check config.toml and settings on the server before scanning +## Step7. Check config.toml and settings on the server before scanning ``` $ vuls configtest ``` 詳細は [Usage: configtest](#usage-configtest) を参照 -## Step7. Start Scanning +## Step8. Start Scanning ``` $ vuls scan + ... snip ... -Scan Summary -============ -localhost amazon 2015.09 94 CVEs 103 updatable packages +One Line Summary +================ +localhost centos7.3.1611 31 updatable packages ``` -## Step8. Reporting +## Step9. Reporting View one-line summary ``` -$ vuls report -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 +$ vuls report -lang=ja -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 -ovaldb-path=$PWD/oval.sqlite3 One Line Summary ================ -localhost Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages +localhost Total: 101 (High:35 Medium:50 Low:16 ?:0) 31 updatable packages ``` View short summary. ``` -$ vuls report -format-short-text -cvedb-path=$PWD/cve.sqlite3 --lang=ja +$ vuls report -lang=ja -format-short-text |less -localhost (amazon 2015.09) -=========================== -Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages +localhost (centos7.3.1611) +========================== +Total: 101 (High:35 Medium:50 Low:16 ?:0) 31 updatable packages -CVE-2016-5636 10.0 (High) CPython の zipimport.c の get_data 関数における整数オーバーフローの脆弱性 - http://jvndb.jvn.jp/ja/contents/2016/JVNDB-2016-004528.html - https://access.redhat.com/security/cve/CVE-2016-5636 - python27-2.7.10-4.119.amzn1 -> python27-2.7.12-2.120.amzn1 - python27-devel-2.7.10-4.119.amzn1 -> python27-devel-2.7.12-2.120.amzn1 - python27-libs-2.7.10-4.119.amzn1 -> python27-libs-2.7.12-2.120.amzn1 - Confidence: 100 / YumUpdateSecurityMatch +CVE-2017-7895 10.0 HIGH (nvd) + Linux Kernel の NFSv2/NFSv3 + サーバの実装におけるポインタ演算エラーを誘発される脆弱性 + Linux Kernel の NFSv2/NFSv3 + サーバの実装は、バッファの終端に対する特定のチェックが欠落しているため、ポイン... + (pointer-arithmetic error) + を誘発されるなど、不特定の影響を受ける脆弱性が存在します。 + --- + http://jvndb.jvn.jp/ja/contents/2017/JVNDB-2017-003674.html + https://access.redhat.com/security/cve/CVE-2017-7895 (RHEL-CVE) + 10.0/AV:N/AC:L/Au:N/C:C/I:C/A:C (nvd) + 10.0/AV:N/AC:L/Au:N/C:C/I:C/A:C (jvn) + https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=CVE-2017-7895 + 6.5/CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N (redhat) + https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=CVE-2017-7895 + Confidence: 100 / OvalMatch -... snip ... ```` View full report. ``` -$ vuls report -format-full-text -cvedb-path=$PWD/cve.sqlite3 --lang=ja +$ vuls report -lang=ja -format-full-text |less -localhost (amazon 2015.09) -============================ -Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages +localhost (centos7.3.1611) +========================== +Total: 101 (High:35 Medium:50 Low:16 ?:0) 31 updatable packages -CVE-2016-5636 -------------- -Score 10.0 (High) -Vector (AV:N/AC:L/Au:N/C:C/I:C/A:C) -Title CPython の zipimport.c の get_data 関数における整数オーバーフローの脆弱性 -Description CPython (別名 Python) の zipimport.c の get_data - 関数には、整数オーバーフローの脆弱性が存在します。 +CVE-2015-2806 +---------------- +Max Score 10.0 HIGH (nvd) +nvd 10.0/AV:N/AC:L/Au:N/C:C/I:C/A:C +redhat 2.6/AV:N/AC:H/Au:N/C:N/I:N/A:P +redhat 3.3/CVSS:3.0/AV:L/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L +CVSSv2 Calc https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=CVE-2015-2806 +CVSSv3 Calc https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=CVE-2015-2806 +Summary Stack-based buffer overflow in asn1_der_decoding in libtasn1 before 4.4 allows + remote attackers to have unspecified impact via unknown vectors. +Source https://nvd.nist.gov/vuln/detail/CVE-2015-2806 +RHEL-CVE https://access.redhat.com/security/cve/CVE-2015-2806 +CWE-119 (nvd) https://cwe.mitre.org/data/definitions/119.html +Package/CPE libtasn1-3.8-3.el7 - +Confidence 100 / OvalMatch - 補足情報 : CWE による脆弱性タイプは、CWE-190: Integer Overflow or Wraparound - (整数オーバーフローまたはラップアラウンド) と識別されています。 - http://cwe.mitre.org/data/definitions/190.html -CWE-190 https://cwe.mitre.org/data/definitions/190.html -CWE-190(JVN) http://jvndb.jvn.jp/ja/cwe/CWE-190.html -JVN http://jvndb.jvn.jp/ja/contents/2016/JVNDB-2016-004528.html -NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-5636 -MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-5636 -CVE Details http://www.cvedetails.com/cve/CVE-2016-5636 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-5636&vector=(AV:N/AC:L/... -RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-5636 -ALAS-2016-724 https://alas.aws.amazon.com/ALAS-2016-724.html -Package python27-2.7.10-4.119.amzn1 -> python27-2.7.12-2.120.amzn1 - python27-devel-2.7.10-4.119.amzn1 -> python27-devel-2.7.12-2.120.amzn1 - python27-libs-2.7.10-4.119.amzn1 -> python27-libs-2.7.12-2.120.amzn1 -Confidence 100 / YumUpdateSecurityMatch ... snip ... ``` -## Step9. TUI +## Step10. TUI Vulsにはスキャン結果の詳細を参照できるイカしたTUI(Terminal-Based User Interface)が付属している。 @@ -385,7 +456,7 @@ $ vuls tui ![Vuls-TUI](img/hello-vuls-tui.png) -## Step10. Web UI +## Step11. Web UI [VulsRepo](https://github.com/usiusi360/vulsrepo)はスキャン結果をビボットテーブルのように分析可能にするWeb UIである。 [Online Demo](http://usiusi360.github.io/vulsrepo/)があるので試してみて。 @@ -396,33 +467,28 @@ $ vuls tui SSHを用いてリモートのホストをスキャンする方法を説明する。 -1. Amazon Linuxを新規に1台作成（スキャン対象） -1. 必要なソフトウェアをインストール -1. RemoteホストにlocalhostからSSH可能にする -1. 設定 +1. Ubuntu Linuxを新規に1台作成（スキャン対象） +1. スキャン対象のRemoteホストにlocalhostからSSH可能にする +1. config.tomlの設定 1. 設定ファイルと、スキャン対象サーバの設定のチェック 1. Scan 1. Reporting 先程のチュートリアルで作成したVulsサーバ(以下localhostと記述)を用いる。 -## Step1. Launch Another Amazon Linux +## Step1. Launch new Ubuntu Linux (the server to be sacnned) [Tutorial: Local Scan Mode#Step1. Launch Amazon Linux](#step1-launch-amazon-linux)と同じ +[Tutorial: Local Scan Mode#Step1. Launch CentOS7](#step1-launch-centos7)のようにUbuntu Linuxを新規に作成する。 新規にターミナルを開いて今作成したEC2にSSH接続する。 +$HOME/.ssh/known_hostsにリモートホストのHost Keyを追加するために、スキャン前にリモートホストにSSH接続する必要がある。 -## Step2. Install Dependencies on the Remote Server - -ディストリビューションによってはスキャンに必要な依存ソフトウェアをインストールする必要がある。 -これらはリモートサーバ上に手動かAnsibleなどでインストールする。 -依存ソフトウェアの詳細は [Dependencies on Target Servers](#dependencies-on-target-servers) を参照。 - -## Step3. Enable to SSH from Localhost +## Step2. Enable to SSH from localhost VulsはSSHパスワード認証をサポートしてない。SSHの鍵認証の設定をしなければならない。 localhost上でkeypairを作成し、remote host上のauthorized_keysに追加する。 -- Localhost +- localhost ```bash $ ssh-keygen -t rsa ``` @@ -436,53 +502,56 @@ $ touch ~/.ssh/authorized_keys $ chmod 600 ~/.ssh/authorized_keys $ vim ~/.ssh/authorized_keys ``` -Paste from the clipboard to ~/.ssh/.authorized_keys +Paste from the clipboard to `~/.ssh/.authorized_keys` -パスワードありのsudoはセキュリティ上の理由からサポートしていないので、スキャンに必要なコマンドは、`NOPASSAWORD`として、remote host上の`etc/sudoers`に定義しておく。 -See [Usage: Configtest#Check /etc/sudoers](#check-etcsudoers) +localhostのknown_hostsにremote hostのホストキーが登録されている必要があるので確認すること。 +`$HOME/.ssh/known_hosts`にリモートホストのHost Keyを追加するために、スキャン前にリモートホストにSSH接続する必要がある。 -また、localhostのknown_hostsにremote hostのホストキーが登録されている必要があるので確認すること。 -## Step4. Config +- localhost +``` +$ ssh ubuntu@172.31.4.82 -i ~/.ssh/id_rsa +``` -- Localhost +## Step3. config.tomlの設定 + +- localhost ``` $ cd $HOME $ cat config.toml [servers] -[servers.172-31-4-82] +[servers.ubuntu] host = "172.31.4.82" port = "22" -user = "ec2-user" -keyPath = "/home/ec2-user/.ssh/id_rsa" +user = "ubuntu" +keyPath = "/home/centos/.ssh/id_rsa" ``` -## Step5. Check config.toml and settings on the server before scanning +## Step4. Check config.toml and settings on the server before scanning ``` -$ vuls configtest +$ vuls configtest ubuntu ``` see [Usage: configtest](#usage-configtest) -## Step6. Start Scanning +## Step5. Start Scanning ``` -$ vuls scan +$ vuls scan ubuntu ... snip ... -Scan Summary -============ -172-31-4-82 amazon 2015.09 94 CVEs 103 updatable packages - +One Line Summary +================ +ubuntu ubuntu16.04 30 updatable packages ``` -## Step7. Reporting +## Step6. Reporting -See [Tutorial: Local Scan Mode#Step8. Reporting](#step8-reporting) -See [Tutorial: Local Scan Mode#Step9. TUI](#step9-tui) -See [Tutorial: Local Scan Mode#Step10. Web UI](#step10-web-ui) +See [Tutorial: Local Scan Mode#Step9. Reporting](#step9-reporting) +See [Tutorial: Local Scan Mode#Step10. TUI](#step10-tui) +See [Tutorial: Local Scan Mode#Step11. Web UI](#step11-web-ui) ---- @@ -500,20 +569,45 @@ Vulsをスキャン対象サーバにデプロイする。Vulsはローカルホ ![Vuls-Architecture Local Scan Mode](img/vuls-architecture-localscan.png) [詳細](#example-scan-via-shell-instead-of-ssh) -## [go-cve-dictionary](https://github.com/kotakanbe/go-cve-dictionary) -- NVDとJVN(日本語)から脆弱性データベースを取得し、SQLite3に格納する。 +----- -## Vuls -![Vuls-Scan-Flow](img/vuls-scan-flow.png) -- SSHでサーバに存在する脆弱性をスキャンし、CVE IDのリストを作成する - - Dockerコンテナのスキャンする場合、VulsはまずDockerホストにSSHで接続する。その後、Dockerホスト上で `docker exec` 経由でコマンドを実効する。Dockerコンテナ内にSSHデーモンを起動する必要はない -- 検出されたCVEの詳細情報をgo-cve-dictionaryから取得する -- スキャン結果レポートを生成し、SlackやEmailなどで送信する -- スキャン結果をJSONファイルに出力すると詳細情報をターミナル上で参照可能 +## Fast Scan and Deep Scan +### Fast Scan +![Vuls-Scan-Flow](img/vuls-scan-flow-fast.png) +- Root権限不要でスキャン可能なモード（Raspbian以外） +- OVALが提供されているディストリビューションは、スキャン時はパッケージのバージョンを取得するのみ。レポート時にOVAL DBとバージョン比較により脆弱性を検知する +- OVALが提供されいていないディストリビューションはスキャン時にコマンドを発行して脆弱性を検知する + +| Distribution| Scan Speed | Need Root Privilege | OVAL | Need Internet Access
on scan tareget| +|:------------|:--------------------------------------:|:-------------------:|:----------:|:---------------------------------------:| +| CentOS | Fast |　 No | Supported | No | +| RHEL | Fast |　 No | Supported | No | +| Oracle | Fast |　 No | Supported | No | +| Ubuntu | Fast |　 No | Supported | No | +| Debian | Fast |　 No | Supported | No | +| FreeBSD | Fast |　 No | No | Need | +| Amazon | Fast |　 No | No | Need | +| Raspbian |1st time: Slow
From 2nd time: Fast | Need | No | Need | ---- -# Performance Considerations + +### Deep Scan +![Vuls-Scan-Flow](img/vuls-scan-flow.png) +- Root権限が必要なコマンドも発行し、より深いスキャンを行うモード +- ChangelogをパースしてCVE-IDを検知するのでFastよりも検知漏れが減る + +| Distribution| Scan Speed | Need Root Privilege | OVAL | Need Internet Access
on scan tareget| +|:------------|:-------------------------------------:|:-------------------------:|:---------:|:---------------------------------------:| +| CentOS | Slow |　 No | Supported | Need | +| RHEL | Slow |　 Need | Supported | Need | +| Oracle | Slow |　 Need | Supported | Need | +| Ubuntu |1st time: Slow
From 2nd time: Fast| Need | Supported | Need | +| Debian |1st time: Slow
From 2nd time: Fast| Need | Supported | Need | +| FreeBSD | Fast |　 No | No | Need | +| Amazon | Slow |　 No | No | Need | +| Raspbian |1st time: Slow
From 2nd time: Fast| Need | No | Need | + - Ubuntu, Debian, Raspbian `apt-get changelog`でアップデート対象のパッケージのチェンジログを取得し、含まれるCVE IDをパースする。 @@ -521,20 +615,10 @@ Vulsをスキャン対象サーバにデプロイする。Vulsはローカルホ ただ、２回目以降はキャッシュしたchangelogを使うので速くなる。 - CentOS -アップデート対象すべてのchangelogを一度で取得しパースする。スキャンスピードは速い、サーバリソース消費量は小さい。 +`yum changelog`でアップデート対象のパッケージのチェンジログを取得し、含まれるCVE IDをパースする。 - Amazon, RHEL and FreeBSD -高速にスキャンし、スキャン対象サーバのリソース消費量は小さい。 - -| Distribution| Scan Speed | -|:------------|:-------------------| -| Ubuntu | 初回は遅い / 2回目以降速い　| -| Debian | 初回は遅い / 2回目以降速い　| -| CentOS | 速い | -| Amazon | 速い | -| RHEL | 速い | -| FreeBSD | 速い | -| Raspbian | 初回は遅い / 2回目以降速い　| +`yum changelog`でアップデート対象のパッケージのチェンジログを取得する(パースはしない)。 ---- @@ -557,12 +641,12 @@ web/app server in the same configuration under the load balancer | Distribution| Release | |:------------|-------------------:| | Ubuntu | 12, 14, 16| -| Debian | 7, 8| +| Debian | 7, 8, 9| | RHEL | 5, 6, 7| | CentOS | 6, 7| | Amazon Linux| All| | FreeBSD | 10, 11| -| Raspbian | Wheezy, Jessie | +| Raspbian | Jessie, Stretch | ---- @@ -755,6 +839,7 @@ host = "172.31.4.82" $ vuls configtest --help configtest: configtest + [-deep] [-config=/path/to/config.toml] [-log-dir=/path/to/log] [-ask-key-password] @@ -773,6 +858,8 @@ configtest: Test containers only. Default: Test both of hosts and containers -debug debug mode + -deep + Config test for deep scan mode -http-proxy string http://proxy-url:port (default: empty) -log-dir string @@ -783,30 +870,46 @@ configtest: Timeout(Sec) (default 300) ``` -configtestサブコマンドは以下をチェックする -- config.tomlで定義されたサーバ/コンテナに対してSSH可能かどうか +configtestサブコマンドは、config.tomlで定義されたサーバ/コンテナに対してSSH可能かどうかをチェックする。 + +## Fast Scan Mode + +| Distribution | Release | Requirements | +|:-------------|-------------------:|:-------------| +| Ubuntu | 12, 14, 16| - | +| Debian | 7, 8, 9| reboot-notifier| +| CentOS | 6, 7| - | +| Amazon | All | - | +| RHEL | 5, 6, 7 | - | +| Oracle Linux | 5, 6, 7 | - | +| FreeBSD | 10, 11 | - | +| Raspbian | Jessie, Stretch | - | + +## Deep Scan Mode + +Deep Scan Modeではスキャン対象サーバ上にいくつかの依存パッケージが必要。 +configtestに--deepをつけて実行するとSSH接続に加えて以下もチェックする。 - スキャン対象のサーバ上に依存パッケーがインストールされているか - /etc/sudoers -## Dependencies on Target Servers +### Dependencies and /etc/sudoers on Target Servers -スキャンするためには、下記のパッケージが必要なので、手動かまたはAnsibleなどのツールで事前にインストールする必要がある。 +Deep Scan Modeでスキャンするためには、下記のパッケージが必要なので、手動かまたはAnsibleなどのツールで事前にインストールする必要がある。 -| Distribution| Release | Requirements | -|:------------|-------------------:|:-------------| -| Ubuntu | 12, 14, 16| - | -| Debian | 7, 8| aptitude | -| CentOS | 6, 7| yum-plugin-changelog | -| Amazon | All | - | -| RHEL | 5 | yum-security | -| RHEL | 6, 7 | - | -| FreeBSD | 10 | - | -| Raspbian | Wheezy, Jessie | - | +| Distribution | Release | Requirements | +|:-------------|-------------------:|:-------------| +| Ubuntu | 12, 14, 16| - | +| Debian | 7, 8, 9| aptitude, reboot-notifier | +| CentOS | 6, 7| yum-plugin-changelog, yum-utils | +| Amazon | All | yum-plugin-changelog, yum-utils | +| RHEL | 5 | yum-utils, yum-security, yum-changelog | +| RHEL | 6, 7 | yum-utils, yum-plugin-changelog | +| Oracle Linux | 5 | yum-utils, yum-security, yum-changelog | +| Oracle Linux | 6, 7 | yum-utils, yum-plugin-changelog | +| FreeBSD | 10 | - | +| Raspbian | Wheezy, Jessie | - | -## Check /etc/sudoers - -スキャン対象サーバに対してパスワードなしでSUDO可能な状態か確認する。 -また、requirettyも定義されているか確認する。(--ssh-native-insecureオプションでscanする場合はrequirettyは定義しなくても良い) +また、Deep Scan Modeで利用するコマンドの中にはRoot権限が必要なものものある。configtestサブコマンドでは、スキャン対象サーバに対してそのコマンドがパスワードなしでSUDO可能な状態か確認する。また、requirettyも定義されているかも確認する。(--ssh-native-insecureオプションでscanする場合はrequirettyは定義しなくても良い) ``` Defaults:vuls !requiretty ``` @@ -814,37 +917,25 @@ For details, see [-ssh-native-insecure option](#-ssh-native-insecure-option) スキャン対象サーバ上の`/etc/sudoers`のサンプル -- CentOS +- RHEL 5 / Oracle Linux 5 ``` -vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --changelog --assumeno update * +vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never list-security --security, /usr/bin/yum --color=never info-security Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" ``` -- RHEL 5 +- RHEL 6, 7 / Oracle Linux 6, 7 ``` -vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never list-security --security, /usr/bin/yum --color=never check-update, /usr/bin/yum --color=never info-security +vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never --security updateinfo list updates, /usr/bin/yum --color=never --security updateinfo updates Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" ``` -- RHEL 6, 7 -``` -vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never --security updateinfo list updates, /usr/bin/yum --color=never check-update, /usr/bin/yum --color=never --security updateinfo updates -Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" -``` - -- Debian +- Debian/Ubuntu/Raspbian ``` vuls ALL=(ALL) NOPASSWD: /usr/bin/apt-get update Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" ``` -- Ubuntu/Raspbian -``` -vuls ALL=(ALL) NOPASSWD: /usr/bin/apt-get update -Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" -``` - -- Amazon Linux, FreeBSDは今のところRoot権限なしでスキャン可能 +- CentOS, Amazon Linux, FreeBSDは今のところRoot権限なしでスキャン可能 ---- @@ -854,6 +945,7 @@ Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" $ vuls scan -help scan: scan + [-deep] [-config=/path/to/config.toml] [-results-dir=/path/to/results] [-log-dir=/path/to/log] @@ -879,6 +971,8 @@ scan: Scan containers only. Default: Scan both of hosts and containers -debug debug mode + -deep + Deep scan mode. Scan accuracy improves and information becomes richer. Since analysis of changelog, issue commands requiring sudo, but it may be slower and high load on the scan tareget server. -http-proxy string http://proxy-url:port (default: empty) -log-dir string @@ -897,6 +991,14 @@ scan: Number of second for scaning vulnerabilities for all servers (default 7200) ``` +## -deep option + +You need to execute `vuls configtest --deep` to check the configuration of the target server before scanning with -deep flag. + +For details about deep scan mode, see below. +* [Architecture/Deep Scan](#deep-scan) +* [Configtest/Deep Scan Mode](#deep-scan-mode) + ## -ssh-native-insecure option Vulsは２種類のSSH接続方法をサポートしている。 @@ -1041,9 +1143,12 @@ report: [-results-dir=/path/to/results] [-log-dir=/path/to/log] [-refresh-cve] - [-cvedb-type=sqlite3|mysql|postgres] + [-cvedb-type=sqlite3|mysql|postgres|redis] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 or DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-cvss-over=7] [-diff] [-ignore-unscored-cves] @@ -1062,6 +1167,7 @@ report: [-aws-profile=default] [-aws-region=us-west-2] [-aws-s3-bucket=bucket_name] + [-aws-s3-results-dir=/bucket/path/to/results] [-azure-account=accout] [-azure-key=key] [-azure-container=container] @@ -1077,6 +1183,8 @@ report: AWS region to use (default "us-east-1") -aws-s3-bucket string S3 bucket name + -aws-s3-results-dir string + /bucket/path/to/results (option) -azure-account string Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified -azure-container string @@ -1088,7 +1196,7 @@ report: -cvedb-path string /path/to/sqlite3 (For get cve detail from cve.sqlite3) -cvedb-type string - DB type for fetching CVE dictionary (sqlite3, mysql or postgres) (default "sqlite3") + DB type for fetching CVE dictionary (sqlite3, mysql, postgres or redis) (default "sqlite3") -cvedb-url string http://cve-dictionary.com:8080 or DB connection string -cvss-over float @@ -1121,6 +1229,12 @@ report: [en|ja] (default "en") -log-dir string /path/to/log (default "/var/log/vuls") + -ovaldb-path string + /path/to/sqlite3 (For get oval detail from oval.sqlite3) (default "/Users/kotakanbe/go/src/github.com/future-architect/vuls/oval.sqlite3") + -ovaldb-type string + DB type for fetching OVAL dictionary (sqlite3 or mysql) (default "sqlite3") + -ovaldb-url string + http://goval-dictionary.com:1324 or mysql connection string -pipe Use stdin via PIPE -refresh-cve @@ -1134,7 +1248,7 @@ report: -to-localfile Write report to localfile -to-s3 - Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json/xml/txt) + Write report to S3 (bucket/dir/yyyyMMdd_HHmm/servername.json/xml/txt) -to-slack Send report via Slack ``` @@ -1176,46 +1290,45 @@ Confidence 100 / YumUpdateSecurityMatch ### Summary part ``` -172-31-4-82 (amazon 2015.09) -============================ -Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages +cent6 (centos6.6) +================= +Total: 145 (High:23 Medium:101 Low:21 ?:0) 83 updatable packages ``` -- `172-31-4-82` means that it is a scan report of `servers.172-31-4-82` defined in cocnfig.toml. -- `(amazon 2015.09)` means that the version of the OS is Amazon Linux 2015.09. -- `Total: 94 (High:19 Medium:54 Low:7 ?:14)` means that a total of 94 vulnerabilities exist, and the distribution of CVSS Severity is displayed. -- `103 updatable packages` means that there are 103 updateable packages on the target server. +- `cent6` means that it is a scan report of `servers.cent6` defined in cocnfig.toml. +- `(centos6.6)` means that the version of the OS is CentOS6.6. +- `Total: 145 (High:23 Medium:101 Low:21 ?:0)` means that a total of 145 vulnerabilities exist, and the distribution of CVSS Severity is displayed. +- `83 updatable packages` means that there are 83 updateable packages on the target server. ### Detailed Part ``` -CVE-2016-5636 -------------- -Score 10.0 (High) -Vector (AV:N/AC:L/Au:N/C:C/I:C/A:C) -Summary Integer overflow in the get_data function in zipimport.c in CPython (aka Python) - before 2.7.12, 3.x before 3.4.5, and 3.5.x before 3.5.2 allows remote attackers - to have unspecified impact via a negative data size value, which triggers a - heap-based buffer overflow. -CWE https://cwe.mitre.org/data/definitions/190.html -NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-5636 -MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-5636 -CVE Details http://www.cvedetails.com/cve/CVE-2016-5636 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-5636&vector=(AV:N/AC:L/... -RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-5636 -ALAS-2016-724 https://alas.aws.amazon.com/ALAS-2016-724.html -Package python27-2.7.10-4.119.amzn1 -> python27-2.7.12-2.120.amzn1 - python27-devel-2.7.10-4.119.amzn1 -> python27-devel-2.7.12-2.120.amzn1 - python27-libs-2.7.10-4.119.amzn1 -> python27-libs-2.7.12-2.120.amzn1 -Confidence 100 / YumUpdateSecurityMatch +CVE-2016-0702 +---------------- +Max Score 2.6 IMPORTANT (redhat) +nvd 1.9/AV:L/AC:M/Au:N/C:P/I:N/A:N +redhat 2.6/AV:L/AC:H/Au:N/C:P/I:P/A:N +jvn 1.9/AV:L/AC:M/Au:N/C:P/I:N/A:N +CVSSv2 Calc https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=CVE-2016-0702 +Summary The MOD_EXP_CTIME_COPY_FROM_PREBUF function in crypto/bn/bn_exp.c in OpenSSL + 1.0.1 before 1.0.1s and 1.0.2 before 1.0.2g does not properly consider + cache-bank access times during modular exponentiation, which makes it easier for + local users to discover RSA keys by running a crafted application on the same + Intel Sandy Bridge CPU core as a victim and leveraging cache-bank conflicts, aka + a "CacheBleed" attack. +Source https://nvd.nist.gov/vuln/detail/CVE-2016-0702 +RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0702 +CWE-200 (nvd) https://cwe.mitre.org/data/definitions/200.html +Package/CPE openssl-1.0.1e-30.el6 - 1.0.1e-57.el6 +Confidence 100 / OvalMatch ``` -- `Score` means CVSS Score. -- `Vector` means [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) +- `Max Score` means Max CVSS Score. +- `nvd` shows [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) of NVD +- `redhat` shows [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) of RedHat OVAL +- `jvn` shows [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) of JVN - `Summary` means Summary of the CVE. - `CWE` means [CWE - Common Weakness Enumeration](https://nvd.nist.gov/cwe.cfm) of the CVE. -- `NVD` `MITRE` `CVE Details` `CVSS Caluculator` -- `RHEL-CVE` means the URL of OS distributor support. - `Package` shows the package version information including this vulnerability. - `Confidence` means the reliability of detection. - `100` is highly reliable @@ -1224,34 +1337,14 @@ Confidence 100 / YumUpdateSecurityMatch | Detection Method | Confidence | OS |Description| |:-----------------------|-------------------:|:---------------------------------|:--| - | YumUpdateSecurityMatch | 100 | RHEL, Amazon Linux |Detection using yum-plugin-security| + | OvalMatch | 100 | CentOS, RHEL, Oracle, Ubuntu, Debian |Detection using OVAL | + | YumUpdateSecurityMatch | 100 | RHEL, Amazon, Oracle |Detection using yum-plugin-security| | ChangelogExactMatch | 95 | CentOS, Ubuntu, Debian, Raspbian |Exact version match between changelog and package version| | ChangelogLenientMatch | 50 | Ubuntu, Debian, Raspbian |Lenient version match between changelog and package version| | PkgAuditMatch | 100 | FreeBSD |Detection using pkg audit| | CpeNameMatch | 100 | All |Search for NVD information with CPE name specified in config.toml| -### Changelog Part - -The scan results of Ubuntu, Debian, Raspbian or CentOS are also output Changelog in TUI or report with -format-full-text. -(RHEL, Amazon or FreeBSD will be available in the near future) - -The output change log includes only the difference between the currently installed version and candidate version. - -``` -tar-1.28-2.1 -> tar-1.28-2.1ubuntu0.1 -------------------------------------- -tar (1.28-2.1ubuntu0.1) xenial-security; urgency=medium - - * SECURITY UPDATE: extract pathname bypass - - debian/patches/CVE-2016-6321.patch: skip members whose names contain - ".." in src/extract.c. - - CVE-2016-6321 - - -- Marc Deslauriers Thu, 17 Nov 2016 11:06:07 -0500 -``` - - ## Example: Send scan results to Slack ``` $ vuls report \ @@ -1436,6 +1529,14 @@ $ vuls report \ -cvedb-url=""host=myhost user=user dbname=dbname sslmode=disable password=password"" ``` +## Example: Use Redis as a DB storage back-end + +``` +$ vuls report \ + -cvedb-type=redis -cvedb-url="redis://localhost/0" + -ovaldb-type=redis -ovaldb-url="redis://localhost/1" +``` + ---- # Usage: Scan vulnerability of non-OS package @@ -1496,9 +1597,12 @@ VulsとDependency Checkを連携すると以下の利点がある ``` tui: tui - [-cvedb-type=sqlite3|mysql|postgres] + [-cvedb-type=sqlite3|mysql|postgres|redis] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-refresh-cve] [-results-dir=/path/to/results] [-log-dir=/path/to/log] @@ -1509,9 +1613,15 @@ tui: -cvedb-path string /path/to/sqlite3 (For get cve detail from cve.sqlite3) -cvedb-type string - DB type for fetching CVE dictionary (sqlite3, mysql or postgres) (default "sqlite3") + DB type for fetching CVE dictionary (sqlite3, mysql, postgres or redis) (default "sqlite3") -cvedb-url string http://cve-dictionary.com:8080 or DB connection string + -ovaldb-path string + /path/to/sqlite3 (For get oval detail from oval.sqlite3) (default "/Users/kotakanbe/go/src/github.com/future-architect/vuls/oval.sqlite3") + -ovaldb-type string + DB type for fetching OVAL dictionary (sqlite3 or mysql) (default "sqlite3") + -ovaldb-url string + http://goval-dictionary.com:1324 or mysql connection string -debug debug mode -debug-sql @@ -1575,7 +1685,7 @@ $ go-cve-dictionary server -bind=192.168.10.1 -port=1323 Run Vuls with -cve-dictionary-url option. ``` -$ vuls scan -cve-dictionary-url=http://192.168.0.1:1323 +$ vuls report -cve-dictionary-url=http://192.168.0.1:1323 ``` # Usage: Update NVD Data @@ -1584,6 +1694,27 @@ see [go-cve-dictionary#usage-fetch-nvd-data](https://github.com/kotakanbe/go-cve ---- +# Usage: goval-dictionary on different server + +``` +$ goval-dictionary server -bind=192.168.10.1 -port=1324 +``` + +Run Vuls with -ovaldb-url option. + +``` +$ vuls report -ovaldb-url=http://192.168.0.1:1323 +``` + +# Usage: Update OVAL Data + +- [RedHat, CentOS](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-redhat) +- [Ubuntu](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-ubuntu) +- [Debian](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-debian) +- [Oracle](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-oracle) + +---- + # レポートの日本語化 see [go-cve-dictionary#usage-fetch-jvn-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-jvn-data) @@ -1618,14 +1749,23 @@ slack, emailは日本語対応済み TUIは日本語表示未対応 ---- -# Update Vuls With Glide +# How to Update to the Latest Version - Update go-cve-dictionary -If the DB schema was changed, please specify new SQLite3 or MySQL DB file. +If the DB schema was changed, please specify new SQLite3, MySQL, PostgreSQL or Redis DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull -$ mv vendor /tmp/foo +$ rm -r vendor +$ make install +``` + +- Update goval-dictionary +If the DB schema was changed, please specify new SQLite3, MySQL, PostgreSQL or Redis DB file. +``` +$ cd $GOPATH/src/github.com/kotakanbe/goval-dictionary +$ git pull +$ rm -r vendor $ make install ``` @@ -1633,10 +1773,11 @@ $ make install ``` $ cd $GOPATH/src/github.com/future-architect/vuls $ git pull -$ mv vendor /tmp/bar +$ rm -r vendor $ make install ``` - バイナリファイルは`$GOPATH/bin`以下に作成される +- もしエラーが出る場合は `$GOPATH/pkg` を削除してから実行する --- @@ -1730,6 +1871,11 @@ kotakanbe ([@kotakanbe](https://twitter.com/kotakanbe)) created vuls and [these Please see [CHANGELOG](https://github.com/future-architect/vuls/blob/master/CHANGELOG.md). ---- +# Stargazers over time + +[![Stargazers over time](https://starcharts.herokuapp.com/future-architect/vuls.svg)](https://starcharts.herokuapp.com/future-architect/vuls) + +----- # License diff --git a/README.md b/README.md index fb738303..7c40a6f7 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,15 @@ ![Vuls-logo](img/vuls_logo.png) -Vulnerability scanner for Linux/FreeBSD, agentless, written in golang. - +Vulnerability scanner for Linux/FreeBSD, agentless, written in golang. We have a slack team. [Join slack team](http://goo.gl/forms/xm5KFo35tu) +Twitter: [@vuls_en](https://twitter.com/vuls_en) -[README in Japanese](https://github.com/future-architect/vuls/blob/master/README.ja.md) +[README 日本語](https://github.com/future-architect/vuls/blob/master/README.ja.md) [README in French](https://github.com/future-architect/vuls/blob/master/README.fr.md) +![Vuls-Abstract](img/vuls-abstract.png) + [![asciicast](https://asciinema.org/a/3y9zrf950agiko7klg8abvyck.png)](https://asciinema.org/a/3y9zrf950agiko7klg8abvyck) ![Vuls-slack](img/vuls-slack-en.png) @@ -23,88 +25,99 @@ We have a slack team. [Join slack team](http://goo.gl/forms/xm5KFo35tu) # TOC -- [Vuls: VULnerability Scanner](#vuls-vulnerability-scanner) -- [TOC](#toc) -- [Abstract](#abstract) -- [Main Features](#main-features) -- [What Vuls Doesn't Do](#what-vuls-doesnt-do) -- [Setup Vuls](#setup-vuls) -- [Tutorial: Local Scan Mode](#tutorial-local-scan-mode) - * [Step1. Launch Amazon Linux](#step1-launch-amazon-linux) - * [Step2. Install requirements](#step2-install-requirements) - * [Step3. Deploy go-cve-dictionary](#step3-deploy-go-cve-dictionary) - * [Step4. Deploy Vuls](#step4-deploy-vuls) - * [Step5. Config](#step5-config) - * [Step6. Check config.toml and settings on the server before scanning](#step6-check-configtoml-and-settings-on-the-server-before-scanning) - * [Step7. Start Scanning](#step7-start-scanning) - * [Step8. Reporting](#step8-reporting) - * [Step9. TUI](#step9-tui) - * [Step10. Web UI](#step10-web-ui) -- [Tutorial: Remote Scan Mode](#tutorial-remote-scan-mode) - * [Step1. Launch Another Amazon Linux](#step1-launch-another-amazon-linux) - * [Step2. Install Dependencies on the Remote Server](#step2-install-dependencies-on-the-remote-server) - * [Step3. Enable to SSH from Localhost](#step3-enable-to-ssh-from-localhost) - * [Step4. Config](#step4-config) - * [Step5. Check config.toml and settings on the server before scanning](#step5-check-configtoml-and-settings-on-the-server-before-scanning) - * [Step6. Start Scanning](#step6-start-scanning) - * [Step7. Reporting](#step7-reporting) -- [Setup Vuls in a Docker Container](#setup-vuls-in-a-docker-container) -- [Architecture](#architecture) - * [A. Scan via SSH Mode (Remote Scan Mode)](#a-scan-via-ssh-mode-remote-scan-mode) - * [B. Scan without SSH (Local Scan Mode)](#b-scan-without-ssh-local-scan-mode) - * [go-cve-dictionary](#go-cve-dictionary) - * [Scanning Flow](#scanning-flow) -- [Performance Considerations](#performance-considerations) -- [Use Cases](#use-cases) - * [Scan All Servers](#scan-all-servers) - * [Scan a Single Server](#scan-a-single-server) - * [Scan Staging Environment](#scan-staging-environment) -- [Support OS](#support-os) -- [Usage: Automatic Server Discovery](#usage-automatic-server-discovery) - * [Example](#example) -- [Configuration](#configuration) -- [Usage: Configtest](#usage-configtest) - * [Dependencies on Target Servers](#dependencies-on-target-servers) - * [Check /etc/sudoers](#check-etcsudoers) -- [Usage: Scan](#usage-scan) - * [-ssh-native-insecure option](#-ssh-native-insecure-option) - * [-ask-key-password option](#-ask-key-password-option) - * [Example: Scan all servers defined in config file](#example-scan-all-servers-defined-in-config-file) - * [Example: Scan specific servers](#example-scan-specific-servers) - * [Example: Scan via shell instead of SSH.](#example-scan-via-shell-instead-of-ssh) - + [cron](#cron) - * [Example: Scan containers (Docker/LXD)](#example-scan-containers-dockerlxd) - + [Docker](#docker) - + [LXD](#lxd) -- [Usage: Report](#usage-report) - * [How to read a report](#how-to-read-a-report) - + [Example](#example-1) - + [Summary part](#summary-part) - + [Detailed Part](#detailed-part) - + [Changelog Part](#changelog-part) - * [Example: Send scan results to Slack](#example-send-scan-results-to-slack) - * [Example: Put results in S3 bucket](#example-put-results-in-s3-bucket) - * [Example: Put results in Azure Blob storage](#example-put-results-in-azure-blob-storage) - * [Example: IgnoreCves](#example-ignorecves) - * [Example: Add optional key-value pairs to JSON](#example-add-optional-key-value-pairs-to-json) - * [Example: Use MySQL as a DB storage back-end](#example-use-mysql-as-a-db-storage-back-end) - * [Example: Use PostgreSQL as a DB storage back-end](#example-use-postgresql-as-a-db-storage-back-end) -- [Usage: Scan vulnerabilites of non-OS packages](#usage-scan-vulnerabilites-of-non-os-packages) -- [Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental)](#usage-integrate-with-owasp-dependency-check-to-automatic-update-when-the-libraries-are-updated-experimental) -- [Usage: TUI](#usage-tui) - * [Display the latest scan results](#display-the-latest-scan-results) - * [Display the previous scan results](#display-the-previous-scan-results) -- [Display the previous scan results using peco](#display-the-previous-scan-results-using-peco) -- [Usage: go-cve-dictionary on different server](#usage-go-cve-dictionary-on-different-server) -- [Usage: Update NVD Data](#usage-update-nvd-data) -- [How to Update](#how-to-update) -- [Misc](#misc) -- [Related Projects](#related-projects) -- [Data Source](#data-source) -- [Authors](#authors) -- [Contribute](#contribute) -- [Change Log](#change-log) -- [License](#license) +Table of Contents +================= + + * [Vuls: VULnerability Scanner](#vuls-vulnerability-scanner) + * [TOC](#toc) + * [Abstract](#abstract) + * [Main Features](#main-features) + * [What Vuls Doesn't Do](#what-vuls-doesnt-do) + * [Setup Vuls](#setup-vuls) + * [Tutorial](#tutorial) + * [Tutorial: Local Scan Mode](#tutorial-local-scan-mode) + * [Step1. Launch CentOS7](#step1-launch-centos7) + * [Step2. Install requirements](#step2-install-requirements) + * [Step3. Deploy go-cve-dictionary](#step3-deploy-go-cve-dictionary) + * [Step4. Deploy goval-dictionary](#step4-deploy-goval-dictionary) + * [Step5. Deploy Vuls](#step5-deploy-vuls) + * [Step6. Configuration](#step6-configuration) + * [Step7. Check config.toml and settings on the server before scanning](#step7-check-configtoml-and-settings-on-the-server-before-scanning) + * [Step8. Start Scanning](#step8-start-scanning) + * [Step9. Reporting](#step9-reporting) + * [Step10. TUI](#step10-tui) + * [Step11. Web UI](#step11-web-ui) + * [Tutorial: Remote Scan Mode](#tutorial-remote-scan-mode) + * [Step1. Launch new Ubuntu Linux](#step1-launch-new-ubuntu-linux) + * [Step2. Enable to SSH from localhost](#step2-enable-to-ssh-from-localhost) + * [Step3. Configure (config.toml)](#step3-configure-configtoml) + * [Step4. Check config.toml and settings on the server before scanning](#step4-check-configtoml-and-settings-on-the-server-before-scanning) + * [Step5. Start Scanning](#step5-start-scanning) + * [Step6. Reporting](#step6-reporting) + * [Setup Vuls in a Docker Container](#setup-vuls-in-a-docker-container) + * [Architecture](#architecture) + * [A. Scan via SSH Mode (Remote Scan Mode)](#a-scan-via-ssh-mode-remote-scan-mode) + * [B. Scan without SSH (Local Scan Mode)](#b-scan-without-ssh-local-scan-mode) + * [Fast Scan and Deep Scan](#fast-scan-and-deep-scan) + * [Fast Scan](#fast-scan) + * [Deep Scan](#deep-scan) + * [Use Cases](#use-cases) + * [Scan All Servers](#scan-all-servers) + * [Scan a Single Server](#scan-a-single-server) + * [Scan Staging Environment](#scan-staging-environment) + * [Support OS](#support-os) + * [Usage: Automatic Server Discovery](#usage-automatic-server-discovery) + * [Example](#example) + * [Configuration](#configuration) + * [Usage: Configtest](#usage-configtest) + * [Fast Scan Mode](#fast-scan-mode) + * [Deep Scan Mode](#deep-scan-mode) + * [Dependencies and /etc/sudoers on Target Servers](#dependencies-and-etcsudoers-on-target-servers) + * [Usage: Scan](#usage-scan) + * [-deep option](#-deep-option) + * [-ssh-native-insecure option](#-ssh-native-insecure-option) + * [-ask-key-password option](#-ask-key-password-option) + * [Example: Scan all servers defined in config file](#example-scan-all-servers-defined-in-config-file) + * [Example: Scan specific servers](#example-scan-specific-servers) + * [Example: Scan via shell instead of SSH.](#example-scan-via-shell-instead-of-ssh) + * [cron](#cron) + * [Example: Scan containers (Docker/LXD)](#example-scan-containers-dockerlxd) + * [Docker](#docker) + * [LXD](#lxd) + * [Usage: Report](#usage-report) + * [How to read a report](#how-to-read-a-report) + * [Example](#example-1) + * [Summary part](#summary-part) + * [Detailed Part](#detailed-part) + * [Example: Send scan results to Slack](#example-send-scan-results-to-slack) + * [Example: Put results in S3 bucket](#example-put-results-in-s3-bucket) + * [Example: Put results in Azure Blob storage](#example-put-results-in-azure-blob-storage) + * [Example: IgnoreCves](#example-ignorecves) + * [Example: Add optional key-value pairs to JSON](#example-add-optional-key-value-pairs-to-json) + * [Example: Use MySQL as a DB storage back-end](#example-use-mysql-as-a-db-storage-back-end) + * [Example: Use PostgreSQL as a DB storage back-end](#example-use-postgresql-as-a-db-storage-back-end) + * [Example: Use Redis as a DB storage back-end](#example-use-redis-as-a-db-storage-back-end) + * [Usage: Scan vulnerabilites of non-OS packages](#usage-scan-vulnerabilites-of-non-os-packages) + * [Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental)](#usage-integrate-with-owasp-dependency-check-to-automatic-update-when-the-libraries-are-updated-experimental) + * [Usage: TUI](#usage-tui) + * [Display the latest scan results](#display-the-latest-scan-results) + * [Display the previous scan results](#display-the-previous-scan-results) + * [Display the previous scan results using peco](#display-the-previous-scan-results-using-peco) + * [Usage: go-cve-dictionary on different server](#usage-go-cve-dictionary-on-different-server) + * [Usage: Update NVD Data](#usage-update-nvd-data) + * [Usage: goval-dictionary on different server](#usage-goval-dictionary-on-different-server) + * [Usage: Update OVAL Data](#usage-update-oval-data) + * [How to Update to the Latest Version](#how-to-update-to-the-latest-version) + * [Misc](#misc) + * [Related Projects](#related-projects) + * [Data Source](#data-source) + * [Authors](#authors) + * [Contribute](#contribute) + * [Change Log](#change-log) + * [Stargazers over time](#stargazers-over-time) + * [License](#license) + +Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc) ---- @@ -131,15 +144,39 @@ Vuls is a tool created to solve the problems listed above. It has the following # Main Features - Scan for any vulnerabilities in Linux/FreeBSD Server - - Supports Ubuntu, Debian, CentOS, Amazon Linux, RHEL, Oracle Linux, FreeBSD and Raspbian + - Supports FreeBSD, Ubuntu, Debian, CentOS, Amazon Linux, RHEL, Oracle Linux and Raspbian - Cloud, on-premise, Docker +- High quality scan + - Vuls uses Multiple vulnerability databases + - OVAL + - RHSA/ALAS/ELSA/FreeBSD-SA + - Changelog +- Fast scan and Deep scan + - Fast Scan + - Scan without root privilege + - Scan with No internet access. (RedHat, CentOS, OracleLinux, Ubuntu, Debian) + - Almost no load on the scan target server + - Deep Scan + - Scan with root privilege + - Parses the Changelog + Changelog has a history of version changes. When a security issue is fixed, the relevant CVE ID is listed. + By parsing the changelog and analysing the updates between the installed version of software on the server and the newest version of that software + it's possible to create a list of all vulnerabilities that need to be fixed. + - Sometimes load on the scan target server +- Remote scan and Local scan + - Remote Scan + - User is required to only setup one machine that is connected to other target servers via SSH + - Local Scan + - If you don't want the central Vuls server to connect to each server by SSH, you can use Vuls in the Local Scan mode. +- **Dynamic** Analysis + - It is possible to acquire the state of the server by connecting via SSH and executing the command + - Vuls warns when the scan target server was updated the kernel etc. but not restarting it. - Scan middleware that are not included in OS package management - Scan middleware, programming language libraries and framework for vulnerability - Support software registered in CPE -- Agentless architecture - - User is required to only setup one machine that is connected to other target servers via SSH - Nondestructive testing -- Pre-authorization is not necessary　before scanning on AWS +- Pre-authorization is *NOT* necessary before scanning on AWS + - Vuls works well with Continuous Integration since tests can be run every day. This allows you to find vulnerabilities very quickly. - Auto generation of configuration file template - Auto detection of servers set using CIDR, generate configuration file template - Email and Slack notification is possible (supports Japanese language) @@ -167,14 +204,29 @@ Tutorial shows how to setup vuls manually. ---- +# Tutorial + +To give you an idea of how easy Vuls is to use. +This tutorial consists of three steps. +1. Tutorial: Local Scan Mode + - Launch CentOS on AWS + - Deploy Vuls + - Scan localhost, Reporting +1. Tutorial: Remote Scan Mode + - Launch Ubuntu Linux on AWS + - Scan this Ubuntu from the Vuls you set up earlier + +---- + # Tutorial: Local Scan Mode This tutorial will let you scan the vulnerabilities on the localhost with Vuls. This can be done in the following steps. -1. Launch Amazon Linux +1. Launch CentOS 1. Install requirements 1. Deploy go-cve-dictionary +1. Deploy goval-dictionary 1. Deploy Vuls 1. Configuration 1. Check config.toml and settings on the server before scanning @@ -183,9 +235,9 @@ This can be done in the following steps. 1. TUI(Terminal-Based User Interface) 1. Web UI ([VulsRepo](https://github.com/usiusi360/vulsrepo)) -## Step1. Launch Amazon Linux +## Step1. Launch CentOS7 -- We are using the old AMI (amzn-ami-hvm-2015.09.1.x86_64-gp2 - ami-383c1956) for this example +- We are using the old AMI for this example - Add the following to the cloud-init, to avoid auto-update at the first launch. ``` @@ -199,18 +251,18 @@ This can be done in the following steps. Vuls requires the following packages. -- SQLite3 or MySQL +- SQLite3, MySQL, PostgreSQL, Redis - git - gcc - GNU Make -- go v1.7.1 or later (The latest version is recommended) +- go v1.8.3 or later (The latest version is recommended) - https://golang.org/doc/install ```bash -$ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem -$ sudo yum -y install sqlite git gcc make -$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz +$ ssh centos@52.100.100.100 -i ~/.ssh/private.pem +$ sudo yum -y install sqlite git gcc make wget +$ wget https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.8.3.linux-amd64.tar.gz $ mkdir $HOME/go ``` Add these lines into /etc/profile.d/goenv.sh @@ -232,7 +284,7 @@ $ source /etc/profile.d/goenv.sh ```bash $ sudo mkdir /var/log/vuls -$ sudo chown ec2-user /var/log/vuls +$ sudo chown centos /var/log/vuls $ sudo chmod 700 /var/log/vuls $ $ mkdir -p $GOPATH/src/github.com/kotakanbe @@ -242,6 +294,8 @@ $ cd go-cve-dictionary $ make install ``` The binary was built under `$GOPATH/bin` +If the installation process stops halfway, try increasing the instance type of EC2. An out of memory error may have occurred. + Fetch vulnerability data from NVD. It takes about 10 minutes (on AWS). @@ -251,10 +305,38 @@ $ cd $HOME $ for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetchnvd -years $i; done ... snip ... $ ls -alh cve.sqlite3 --rw-r--r-- 1 ec2-user ec2-user 7.0M Mar 24 13:20 cve.sqlite3 +-rw-r--r--. 1 centos centos 51M Aug 6 08:10 cve.sqlite3 +-rw-r--r--. 1 centos centos 32K Aug 6 08:10 cve.sqlite3-shm +-rw-r--r--. 1 centos centos 5.1M Aug 6 08:10 cve.sqlite3-wal ``` -## Step4. Deploy Vuls +## Step4. Deploy goval-dictionary + +[goval-dictionary](https://github.com/kotakanbe/goval-dictionary) + +```bash +$ mkdir -p $GOPATH/src/github.com/kotakanbe +$ cd $GOPATH/src/github.com/kotakanbe +$ git clone https://github.com/kotakanbe/goval-dictionary.git +$ cd goval-dictionary +$ make install +``` +The binary was built under `$GOPATH/bin` +If the installation process stops halfway, try increasing the instance type of EC2. An out of memory error may have occurred. + + Then fetch OVAL data of RedHat since the server to be scanned is CentOS. [README](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-redhat) + +```bash +$ goval-dictionary fetch-redhat 7 +``` + +If you want to scan other than CentOS 7, fetch OVAL data according to the OS type and version of scan target server in advance. +- [RedHat, CentOS](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-redhat) +- [Debian](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-debian) +- [Ubuntu](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-ubuntu) +- [Oracle Linux](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-oracle) + +## Step5. Deploy Vuls Launch a new terminal and SSH to the ec2 instance. @@ -266,8 +348,9 @@ $ cd vuls $ make install ``` The binary was built under `$GOPATH/bin` +If the installation process stops halfway, try increasing the instance type of EC2. An out of memory error may have occurred. -## Step5. Config +## Step6. Configuration Create a config file(TOML format). ``` @@ -276,15 +359,12 @@ $ cat config.toml [servers] [servers.localhost] -host = "localhost" -port = "local" +host = "localhost" +port = "local" ``` -Root privilege is needed on Some distributions. -Sudo with password is not supported for security reasons. So you have to define NOPASSWORD in /etc/sudoers. -See [Usage: Configtest#Check /etc/sudoers](#check-etcsudoers) -## Step6. Check config.toml and settings on the server before scanning +## Step7. Check config.toml and settings on the server before scanning ``` $ vuls configtest @@ -292,86 +372,85 @@ $ vuls configtest see [Usage: configtest](#usage-configtest) -## Step7. Start Scanning +## Step8. Start Scanning ``` $ vuls scan + ... snip ... -Scan Summary -============ -localhost amazon 2015.09 94 CVEs 103 updatable packages +One Line Summary +================ +localhost centos7.3.1611 31 updatable packages ``` -## Step8. Reporting +## Step9. Reporting View one-line summary ``` -$ vuls report -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 +$ vuls report -format-one-line-text -cvedb-path=$PWD/cve.sqlite3 -ovaldb-path=$PWD/oval.sqlite3 One Line Summary ================ -localhost Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages +localhost Total: 109 (High:35 Medium:55 Low:16 ?:3) 31 updatable packages ``` -View short summary. +View short summary ``` $ vuls report -format-short-text -localhost (amazon 2015.09) -=========================== -Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages +localhost (centos7.3.1611) +========================== +Total: 109 (High:35 Medium:55 Low:16 ?:3) 31 updatable packages + +CVE-2015-2806 10.0 HIGH (nvd) + Stack-based buffer overflow in asn1_der_decoding in libtasn1 before 4.4 allows + remote attackers to have unspecified impact via unknown vectors. + --- + https://nvd.nist.gov/vuln/detail/CVE-2015-2806 + https://access.redhat.com/security/cve/CVE-2015-2806 (RHEL-CVE) + 10.0/AV:N/AC:L/Au:N/C:C/I:C/A:C (nvd) + 2.6/AV:N/AC:H/Au:N/C:N/I:N/A:P (redhat) + https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=CVE-2015-2806 + 3.3/CVSS:3.0/AV:L/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L (redhat) + https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=CVE-2015-2806 + Confidence: 100 / OvalMatch -CVE-2016-5636 10.0 (High) Integer overflow in the get_data function in zipimport.c in CPython (aka Python) - before 2.7.12, 3.x before 3.4.5, and 3.5.x before 3.5.2 allows remote attackers - to have unspecified impact via a negative data size value, which triggers a - heap-based buffer overflow. - http://www.cvedetails.com/cve/CVE-2016-5636 - https://access.redhat.com/security/cve/CVE-2016-5636 - python27-2.7.10-4.119.amzn1 -> python27-2.7.12-2.120.amzn1 - python27-devel-2.7.10-4.119.amzn1 -> python27-devel-2.7.12-2.120.amzn1 - python27-libs-2.7.10-4.119.amzn1 -> python27-libs-2.7.12-2.120.amzn1 - Confidence: 100 / YumUpdateSecurityMatch ... snip ... ```` View full report. ``` -$ vuls report -format-full-text +$ vuls report -format-full-text | less +localhost (centos7.3.1611) +========================== +Total: 109 (High:35 Medium:55 Low:16 ?:3) 31 updatable packages -localhost (amazon 2015.09) -============================ -Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages - -CVE-2016-5636 -------------- -Score 10.0 (High) -Vector (AV:N/AC:L/Au:N/C:C/I:C/A:C) -Summary Integer overflow in the get_data function in zipimport.c in CPython (aka Python) - before 2.7.12, 3.x before 3.4.5, and 3.5.x before 3.5.2 allows remote attackers - to have unspecified impact via a negative data size value, which triggers a - heap-based buffer overflow. -CWE https://cwe.mitre.org/data/definitions/190.html -NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-5636 -MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-5636 -CVE Details http://www.cvedetails.com/cve/CVE-2016-5636 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-5636&vector=(AV:N/AC:L/... -RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-5636 -ALAS-2016-724 https://alas.aws.amazon.com/ALAS-2016-724.html -Package python27-2.7.10-4.119.amzn1 -> python27-2.7.12-2.120.amzn1 - python27-devel-2.7.10-4.119.amzn1 -> python27-devel-2.7.12-2.120.amzn1 - python27-libs-2.7.10-4.119.amzn1 -> python27-libs-2.7.12-2.120.amzn1 -Confidence 100 / YumUpdateSecurityMatch +CVE-2015-2806 +---------------- +Max Score 10.0 HIGH (nvd) +nvd 10.0/AV:N/AC:L/Au:N/C:C/I:C/A:C +redhat 2.6/AV:N/AC:H/Au:N/C:N/I:N/A:P +redhat 3.3/CVSS:3.0/AV:L/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L +CVSSv2 Calc https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=CVE-2015-2806 +CVSSv3 Calc https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=CVE-2015-2806 +Summary Stack-based buffer overflow in asn1_der_decoding in libtasn1 before 4.4 allows + remote attackers to have unspecified impact via unknown vectors. +Source https://nvd.nist.gov/vuln/detail/CVE-2015-2806 +RHEL-CVE https://access.redhat.com/security/cve/CVE-2015-2806 +CWE-119 (nvd) https://cwe.mitre.org/data/definitions/119.html +Package/CPE libtasn1-3.8-3.el7 - +Confidence 100 / OvalMatch ... snip ... ``` -## Step9. TUI +## Step10. TUI Vuls has Terminal-Based User Interface to display the scan result. @@ -381,7 +460,7 @@ $ vuls tui ![Vuls-TUI](img/hello-vuls-tui.png) -## Step10. Web UI +## Step11. Web UI [VulsRepo](https://github.com/usiusi360/vulsrepo) is a awesome Web UI for Vuls. Check it out the [Online Demo](http://usiusi360.github.io/vulsrepo/). @@ -393,9 +472,8 @@ Check it out the [Online Demo](http://usiusi360.github.io/vulsrepo/). This tutorial will let you scan the vulnerabilities on the remote host via SSH with Vuls. This can be done in the following steps. -1. Launch Another Amazon Linux -1. Install Dependencies on the Remote Host -1. Enable to SSH from Localhost +1. Launch new Ubuntu Linux +1. Enable to SSH from localhost 1. Configuration 1. Check config.toml and settings on the server before scanning 1. Scan @@ -403,23 +481,18 @@ This can be done in the following steps. We will use the Vuls server (called localhost) created in the previous tutorial. -## Step1. Launch Another Amazon Linux +## Step1. Launch new Ubuntu Linux -Same as [Tutorial: Local Scan Mode#Step1. Launch Amazon Linux](#step1-launch-amazon-linux) -Launch a new terminal and SSH to the Remote Server. +Same like as [Tutorial: Local Scan Mode#Step1. Launch CentOS7](#step1-launch-centos7) +Launch a new terminal and SSH to the Remote host. +To add the remote host's Host Key to $HOME/.ssh/known_hosts, you need to log in to the remote host through SSH before scanning. -## Step2. Install Dependencies on the Remote Server - -Depending on the distribution you need to install dependent modules. -Install these dependencies manually or using Ansible etc. -For details of dependent libraries, see [Dependencies on Target Servers](#dependencies-on-target-servers) - -## Step3. Enable to SSH from Localhost +## Step2. Enable to SSH from localhost Vuls doesn't support SSH password authentication. So you have to use SSH key-based authentication. -Create a keypair on the localhost then append public key to authorized_keys on the remote host. +Create a keypair on the localhost then append the public key to authorized_keys on the remote host. -- Localhost +- localhost ```bash $ ssh-keygen -t rsa ``` @@ -433,53 +506,55 @@ $ touch ~/.ssh/authorized_keys $ chmod 600 ~/.ssh/authorized_keys $ vim ~/.ssh/authorized_keys ``` -Paste from the clipboard to ~/.ssh/.authorized_keys +Paste from the clipboard to `~/.ssh/.authorized_keys` -SUDO with password is not supported for security reasons. So you have to define NOPASSWORD in /etc/sudoers on target servers. -See [Usage: Configtest#Check /etc/sudoers](#check-etcsudoers) +And also, confirm that the host keys of scan target servers has been registered in the known_hosts of the localhost. +To add the remote host's Host Key to `$HOME/.ssh/known_hosts`, you need to log in to the remote host through SSH before scanning. -And also, confirm that the host keys of scan target servers has been registered in the known_hosts of the Localhost. +- localhost +``` +$ ssh ubuntu@172.31.4.82 -i ~/.ssh/id_rsa +``` -## Step4. Config +## Step3. Configure (config.toml) -- Localhost +- localhost ``` $ cd $HOME $ cat config.toml [servers] -[servers.172-31-4-82] +[servers.ubuntu] host = "172.31.4.82" port = "22" -user = "ec2-user" -keyPath = "/home/ec2-user/.ssh/id_rsa" +user = "ubuntu" +keyPath = "/home/centos/.ssh/id_rsa" ``` -## Step5. Check config.toml and settings on the server before scanning +## Step4. Check config.toml and settings on the server before scanning ``` -$ vuls configtest +$ vuls configtest ubuntu ``` see [Usage: configtest](#usage-configtest) -## Step6. Start Scanning +## Step5. Start Scanning ``` -$ vuls scan +$ vuls scan ubuntu ... snip ... -Scan Summary -============ -172-31-4-82 amazon 2015.09 94 CVEs 103 updatable packages - +One Line Summary +================ +ubuntu ubuntu16.04 30 updatable packages ``` -## Step7. Reporting +## Step6. Reporting -See [Tutorial: Local Scan Mode#Step8. Reporting](#step8-reporting) -See [Tutorial: Local Scan Mode#Step9. TUI](#step9-tui) -See [Tutorial: Local Scan Mode#Step10. Web UI](#step10-web-ui) +See [Tutorial: Local Scan Mode#Step9. Reporting](#step9-reporting) +See [Tutorial: Local Scan Mode#Step10. TUI](#step10-tui) +See [Tutorial: Local Scan Mode#Step11. Web UI](#step11-web-ui) ---- @@ -503,16 +578,43 @@ On the aggregation server, you can refer to the scanning result of each scan tar ![Vuls-Architecture Local Scan Mode](img/vuls-architecture-localscan.png) [Details](#example-scan-via-shell-instead-of-ssh) -## [go-cve-dictionary](https://github.com/kotakanbe/go-cve-dictionary) -- Fetch vulnerability information from NVD and JVN(Japanese), then insert into SQLite3 or MySQL. - -## Scanning Flow -![Vuls-Scan-Flow](img/vuls-scan-flow.png) -- Scan vulnerabilities on the servers via SSH and collect a list of the CVE ID - - To scan Docker containers, Vuls connects via SSH to the Docker host and then `docker exec` to the containers. So, no need to run sshd daemon on the containers. - ---- -# Performance Considerations + +## Fast Scan and Deep Scan + +### Fast Scan +![Vuls-Scan-Flow](img/vuls-scan-flow-fast.png) +- Scan without Root Privilege +- Scan with No internet access on some OS. + +| Distribution| Scan Speed | Need Root Privilege | OVAL | Need Internet Access
on scan tareget| +|:------------|:--------------------------------------:|:-------------------:|:----------:|:---------------------------------------:| +| CentOS | Fast |　 No | Supported | No | +| RHEL | Fast |　 No | Supported | No | +| Oracle | Fast |　 No | Supported | No | +| Ubuntu | Fast |　 No | Supported | No | +| Debian | Fast |　 No | Supported | No | +| Raspbian |1st time: Slow
From 2nd time: Fast | Need | No | Need | +| FreeBSD | Fast |　 No | No | Need | +| Amazon | Fast |　 No | No | Need | + + +--------- + +### Deep Scan +![Vuls-Scan-Flow](img/vuls-scan-flow.png) + +| Distribution| Scan Speed | Need Root Privilege | OVAL | Need Internet Access
on scan tareget| +|:------------|:-------------------------------------:|:-------------------------:|:---------:|:---------------------------------------:| +| CentOS | Slow |　 No | Supported | Need | +| RHEL | Slow |　 Need | Supported | Need | +| Oracle | Slow |　 Need | Supported | Need | +| Ubuntu |1st time: Slow
From 2nd time: Fast| Need | Supported | Need | +| Debian |1st time: Slow
From 2nd time: Fast| Need | Supported | Need | +| Raspbian |1st time: Slow
From 2nd time: Fast| Need | No | Need | +| FreeBSD | Fast |　 No | No | Need | +| Amazon | Slow |　 No | No | Need | + - On Ubuntu, Debian and Raspbian Vuls issues `apt-get changelog` for each upgradable packages and parse the changelog. @@ -520,23 +622,10 @@ Vuls issues `apt-get changelog` for each upgradable packages and parse the chang Vuls stores these changelogs to KVS([boltdb](https://github.com/boltdb/bolt)). From the second time on, the scan speed is fast by using the local cache. -- On CentOS -Vuls issues `yum update --changelog` to get changelogs of upgradable packages at once and parse the changelog. -Scan speed is fast and resource usage is light. - -- On Amazon, RHEL and FreeBSD -High speed scan and resource usage is light because Vuls can get CVE IDs by using package manager(no need to parse a changelog). - -| Distribution | Scan Speed | -|:-------------|:-------------------| -| Ubuntu | First time: Slow / From the second time: Fast | -| Debian | First time: Slow / From the second time: Fast | -| CentOS | Fast | -| Amazon | Fast | -| RHEL | Fast | -| Oracle Linux | Fast | -| FreeBSD | Fast | -| Raspbian | First time: Slow / From the second time: Fast | +- On CentOS +Vuls issues `yum changelog` to get changelogs of upgradable packages at once and parse the changelog. +- On RHEL, Oracle, Amazon and FreeBSD +Detect CVE IDs by using package manager. ---- @@ -563,13 +652,13 @@ If there is a staging environment with the same configuration as the production | Distribution | Release | |:-------------|-------------------:| | Ubuntu | 12, 14, 16| -| Debian | 7, 8| +| Debian | 7, 8, 9| | RHEL | 5, 6, 7| | Oracle Linux | 5, 6, 7| | CentOS | 6, 7| | Amazon Linux | All| | FreeBSD | 10, 11| -| Raspbian | Wheezy, Jessie | +| Raspbian | Jessie, Stretch | ---- @@ -761,6 +850,7 @@ You can customize your configuration using this template. $ vuls configtest --help configtest: configtest + [-deep] [-config=/path/to/config.toml] [-log-dir=/path/to/log] [-ask-key-password] @@ -778,6 +868,8 @@ configtest: Test containers only. Default: Test both of hosts and containers -debug debug mode + -deep + Config test for deep scan mode -http-proxy string http://proxy-url:port (default: empty) -log-dir string @@ -789,31 +881,44 @@ configtest: ``` -The configtest subcommand checks the following -- Whether vuls is able to connect via SSH to servers/containers defined in the config.toml -- Whether Dependent package is installed on the scan target server -- Check /etc/sudoers +The configtest subcommand checks whether vuls is able to connect via SSH to servers/containers defined in the config.toml -## Dependencies on Target Servers + ## Fast Scan Mode -In order to scan, the following dependencies are required, so you need to install them manually or with tools such as Ansible. +| Distribution | Release | Requirements | +|:-------------|-------------------:|:-------------| +| Ubuntu | 12, 14, 16| - | +| Debian | 7, 8, 9| reboot-notifier| +| CentOS | 6, 7| - | +| Amazon | All | - | +| RHEL | 5, 6, 7 | - | +| Oracle Linux | 5, 6, 7 | - | +| FreeBSD | 10, 11 | - | +| Raspbian | Jessie, Stretch | - | + +## Deep Scan Mode + +Some dependent packages are needed in Deep Scan Mode. +The configtest subcommand with --deep flag checks whether the packages are installed on the scan target server and also check /etc/sudoers + +### Dependencies and /etc/sudoers on Target Servers + +In order to scan with deep scan mode, the following dependencies are required, so you need to install them manually or with tools such as Ansible. | Distribution | Release | Requirements | |:-------------|-------------------:|:-------------| | Ubuntu | 12, 14, 16| - | -| Debian | 7, 8| aptitude | -| CentOS | 6, 7| yum-plugin-changelog | -| Amazon | All | - | -| RHEL | 5 | yum-security | -| RHEL | 6, 7 | - | -| Oracle Linux | 5 | yum-security | -| Oracle Linux | 6, 7 | - | +| Debian | 7, 8, 9| aptitude, reboot-notifier | +| CentOS | 6, 7| yum-plugin-changelog, yum-utils | +| Amazon | All | yum-plugin-changelog, yum-utils | +| RHEL | 5 | yum-utils, yum-security, yum-changelog | +| RHEL | 6, 7 | yum-utils, yum-plugin-changelog | +| Oracle Linux | 5 | yum-utils, yum-security, yum-changelog | +| Oracle Linux | 6, 7 | yum-utils, yum-plugin-changelog | | FreeBSD | 10 | - | | Raspbian | Wheezy, Jessie | - | -## Check /etc/sudoers - -The configtest subcommand checks sudo settings on target servers whether Vuls is able to SUDO with nopassword via SSH. And if you run Vuls without -ssh-native-insecure option, requiretty must be defined in /etc/sudoers. +The configtest subcommand also checks sudo settings on target servers whether Vuls is able to SUDO with nopassword via SSH. And if you run Vuls without -ssh-native-insecure option, requiretty must be defined in /etc/sudoers. ``` Defaults:vuls !requiretty ``` @@ -821,37 +926,25 @@ For details, see [-ssh-native-insecure option](#-ssh-native-insecure-option) Example of /etc/sudoers on target servers -- CentOS -``` -vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --changelog --assumeno update * -Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" -``` - - RHEL 5 / Oracle Linux 5 ``` -vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never list-security --security, /usr/bin/yum --color=never check-update, /usr/bin/yum --color=never info-security +vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never list-security --security, /usr/bin/yum --color=never info-security Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" ``` - RHEL 6, 7 / Oracle Linux 6, 7 ``` -vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never --security updateinfo list updates, /usr/bin/yum --color=never check-update, /usr/bin/yum --color=never --security updateinfo updates +vuls ALL=(ALL) NOPASSWD:/usr/bin/yum --color=never repolist, /usr/bin/yum --color=never --security updateinfo list updates, /usr/bin/yum --color=never --security updateinfo updates Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" ``` -- Debian +- Debian/Ubuntu/Raspbian ``` vuls ALL=(ALL) NOPASSWD: /usr/bin/apt-get update Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" ``` -- Ubuntu/Raspbian -``` -vuls ALL=(ALL) NOPASSWD: /usr/bin/apt-get update -Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" -``` - -- On Amazon Linux, FreeBSD, it is possible to scan without root privilege for now. +- On CentOS, Amazon Linux, FreeBSD, it is possible to scan without root privilege for now. ---- @@ -861,6 +954,7 @@ Defaults:vuls env_keep="http_proxy https_proxy HTTP_PROXY HTTPS_PROXY" $ vuls scan -help scan: scan + [-deep] [-config=/path/to/config.toml] [-results-dir=/path/to/results] [-log-dir=/path/to/log] @@ -886,6 +980,8 @@ scan: Scan containers only. Default: Scan both of hosts and containers -debug debug mode + -deep + Deep scan mode. Scan accuracy improves and information becomes richer. Since analysis of changelog, issue commands requiring sudo, but it may be slower and high load on the scan tareget server. -http-proxy string http://proxy-url:port (default: empty) -log-dir string @@ -904,6 +1000,14 @@ scan: Number of second for scaning vulnerabilities for all servers (default 7200) ``` +## -deep option + +You need to execute `vuls configtest --deep` to check the configuration of the target server before scanning with -deep flag. + +For details about deep scan mode, see below. +* [Architecture/Deep Scan](#deep-scan) +* [Configtest/Deep Scan Mode](#deep-scan-mode) + ## -ssh-native-insecure option Vuls supports different types of SSH. @@ -1053,6 +1157,9 @@ report: [-cvedb-type=sqlite3|mysql|postgres] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-cvss-over=7] [-diff] [-ignore-unscored-cves] @@ -1071,6 +1178,7 @@ report: [-aws-profile=default] [-aws-region=us-west-2] [-aws-s3-bucket=bucket_name] + [-aws-s3-results-dir=/bucket/path/to/results] [-azure-account=accout] [-azure-key=key] [-azure-container=container] @@ -1086,6 +1194,8 @@ report: AWS region to use (default "us-east-1") -aws-s3-bucket string S3 bucket name + -aws-s3-results-dir string + /bucket/path/to/results (option) -azure-account string Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified -azure-container string @@ -1130,6 +1240,12 @@ report: [en|ja] (default "en") -log-dir string /path/to/log (default "/var/log/vuls") + -ovaldb-path string + /path/to/sqlite3 (For get oval detail from oval.sqlite3) (default "/Users/kotakanbe/go/src/github.com/future-architect/vuls/oval.sqlite3") + -ovaldb-type string + DB type for fetching OVAL dictionary (sqlite3 or mysql) (default "sqlite3") + -ovaldb-url string + http://goval-dictionary.com:1324 or mysql connection string -pipe Use stdin via PIPE -refresh-cve @@ -1143,7 +1259,7 @@ report: -to-localfile Write report to localfile -to-s3 - Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json/xml/txt) + Write report to S3 (bucket/dir/yyyyMMdd_HHmm/servername.json/xml/txt) -to-slack Send report via Slack ``` @@ -1185,47 +1301,45 @@ Confidence 100 / YumUpdateSecurityMatch ### Summary part ``` -172-31-4-82 (amazon 2015.09) -============================ -Total: 94 (High:19 Medium:54 Low:7 ?:14) 103 updatable packages +cent6 (centos6.6) +================= +Total: 145 (High:23 Medium:101 Low:21 ?:0) 83 updatable packages ``` -- `172-31-4-82` means that it is a scan report of `servers.172-31-4-82` defined in cocnfig.toml. -- `(amazon 2015.09)` means that the version of the OS is Amazon Linux 2015.09. -- `Total: 94 (High:19 Medium:54 Low:7 ?:14)` means that a total of 94 vulnerabilities exist, and the distribution of CVSS Severity is displayed. -- `103 updatable packages` means that there are 103 updateable packages on the target server. +- `cent6` means that it is a scan report of `servers.cent6` defined in cocnfig.toml. +- `(centos6.6)` means that the version of the OS is CentOS6.6. +- `Total: 145 (High:23 Medium:101 Low:21 ?:0)` means that a total of 145 vulnerabilities exist, and the distribution of CVSS Severity is displayed. +- `83 updatable packages` means that there are 83 updateable packages on the target server. ### Detailed Part ``` -CVE-2016-5636 -------------- -Score 10.0 (High) -Vector (AV:N/AC:L/Au:N/C:C/I:C/A:C) -Summary Integer overflow in the get_data function in zipimport.c in CPython (aka Python) - before 2.7.12, 3.x before 3.4.5, and 3.5.x before 3.5.2 allows remote attackers - to have unspecified impact via a negative data size value, which triggers a - heap-based buffer overflow. -CWE https://cwe.mitre.org/data/definitions/190.html -NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-5636 -MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-5636 -CVE Details http://www.cvedetails.com/cve/CVE-2016-5636 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-5636&vector=(AV:N/AC:L/... -RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-5636 -ALAS-2016-724 https://alas.aws.amazon.com/ALAS-2016-724.html -Package python27-2.7.10-4.119.amzn1 -> python27-2.7.12-2.120.amzn1 - python27-devel-2.7.10-4.119.amzn1 -> python27-devel-2.7.12-2.120.amzn1 - python27-libs-2.7.10-4.119.amzn1 -> python27-libs-2.7.12-2.120.amzn1 -Confidence 100 / YumUpdateSecurityMatch +CVE-2016-0702 +---------------- +Max Score 2.6 IMPORTANT (redhat) +nvd 1.9/AV:L/AC:M/Au:N/C:P/I:N/A:N +redhat 2.6/AV:L/AC:H/Au:N/C:P/I:P/A:N +jvn 1.9/AV:L/AC:M/Au:N/C:P/I:N/A:N +CVSSv2 Calc https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=CVE-2016-0702 +Summary The MOD_EXP_CTIME_COPY_FROM_PREBUF function in crypto/bn/bn_exp.c in OpenSSL + 1.0.1 before 1.0.1s and 1.0.2 before 1.0.2g does not properly consider + cache-bank access times during modular exponentiation, which makes it easier for + local users to discover RSA keys by running a crafted application on the same + Intel Sandy Bridge CPU core as a victim and leveraging cache-bank conflicts, aka + a "CacheBleed" attack. +Source https://nvd.nist.gov/vuln/detail/CVE-2016-0702 +RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0702 +CWE-200 (nvd) https://cwe.mitre.org/data/definitions/200.html +Package/CPE openssl-1.0.1e-30.el6 - 1.0.1e-57.el6 +Confidence 100 / OvalMatch ``` -- `Score` means CVSS Score. -- `Vector` means [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) +- `Max Score` means Max CVSS Score. +- `nvd` shows [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) of NVD +- `redhat` shows [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) of RedHat OVAL +- `jvn` shows [CVSS Vector](https://nvd.nist.gov/CVSS/Vector-v2.aspx) of JVN - `Summary` means Summary of the CVE. - `CWE` means [CWE - Common Weakness Enumeration](https://nvd.nist.gov/cwe.cfm) of the CVE. -- `NVD` `MITRE` `CVE Details` `CVSS Caluculator` -- `RHEL-CVE` means the URL of OS distributor support. -- `Oracle-CVE` means the URL of the Oracle Linux errata information. - `Package` shows the package version information including this vulnerability. - `Confidence` means the reliability of detection. - `100` is highly reliable @@ -1234,33 +1348,14 @@ Confidence 100 / YumUpdateSecurityMatch | Detection Method | Confidence | OS |Description| |:-----------------------|-------------------:|:---------------------------------|:--| - | YumUpdateSecurityMatch | 100 | RHEL, Oracle Linux, Amazon Linux |Detection using yum-plugin-security| + | OvalMatch | 100 | CentOS, RHEL, Oracle, Ubuntu, Debian |Detection using OVAL | + | YumUpdateSecurityMatch | 100 | RHEL, Amazon, Oracle |Detection using yum-plugin-security| | ChangelogExactMatch | 95 | CentOS, Ubuntu, Debian, Raspbian |Exact version match between changelog and package version| | ChangelogLenientMatch | 50 | Ubuntu, Debian, Raspbian |Lenient version match between changelog and package version| | PkgAuditMatch | 100 | FreeBSD |Detection using pkg audit| | CpeNameMatch | 100 | All |Search for NVD information with CPE name specified in config.toml| -### Changelog Part - -The scan results of Ubuntu, Debian, Raspbian or CentOS are also output Changelog in TUI or report with -format-full-text. -(RHEL, Amazon or FreeBSD will be available in the near future) - -The output change log includes only the difference between the currently installed version and candidate version. - -``` -tar-1.28-2.1 -> tar-1.28-2.1ubuntu0.1 -------------------------------------- -tar (1.28-2.1ubuntu0.1) xenial-security; urgency=medium - - * SECURITY UPDATE: extract pathname bypass - - debian/patches/CVE-2016-6321.patch: skip members whose names contain - ".." in src/extract.c. - - CVE-2016-6321 - - -- Marc Deslauriers Thu, 17 Nov 2016 11:06:07 -0500 -``` - ## Example: Send scan results to Slack ``` $ vuls report \ @@ -1438,6 +1533,14 @@ $ vuls report \ -cvedb-url=""host=myhost user=user dbname=dbname sslmode=disable password=password"" ``` +## Example: Use Redis as a DB storage back-end + +``` +$ vuls report \ + -cvedb-type=redis -cvedb-url="redis://localhost/0" + -ovaldb-type=redis -ovaldb-url="redis://localhost/1" +``` + ---- # Usage: Scan vulnerabilites of non-OS packages @@ -1499,6 +1602,9 @@ tui: [-cvedb-type=sqlite3|mysql|postgres] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-refresh-cve] [-results-dir=/path/to/results] [-log-dir=/path/to/log] @@ -1512,6 +1618,12 @@ tui: DB type for fetching CVE dictionary (sqlite3, mysql or postgres) (default "sqlite3") -cvedb-url string http://cve-dictionary.com:8080 DB connection string + -ovaldb-path string + /path/to/sqlite3 (For get oval detail from oval.sqlite3) (default "/Users/kotakanbe/go/src/github.com/future-architect/vuls/oval.sqlite3") + -ovaldb-type string + DB type for fetching OVAL dictionary (sqlite3 or mysql) (default "sqlite3") + -ovaldb-url string + http://goval-dictionary.com:1324 or mysql connection string -debug debug mode -debug-sql @@ -1560,6 +1672,8 @@ $ vuls history | peco | vuls tui -pipe [![asciicast](https://asciinema.org/a/emi7y7docxr60bq080z10t7v8.png)](https://asciinema.org/a/emi7y7docxr60bq080z10t7v8) +---- + # Usage: go-cve-dictionary on different server Run go-cve-dictionary as server mode before scanning on 192.168.10.1 @@ -1570,20 +1684,40 @@ $ go-cve-dictionary server -bind=192.168.10.1 -port=1323 Run Vuls with -cvedb-url option. ``` -$ vuls scan -cvedb-url=http://192.168.0.1:1323 +$ vuls report -cvedb-url=http://192.168.0.1:1323 ``` # Usage: Update NVD Data see [go-cve-dictionary#usage-fetch-nvd-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-nvd-data) +---- + +# Usage: goval-dictionary on different server + +``` +$ goval-dictionary server -bind=192.168.10.1 -port=1324 +``` + +Run Vuls with -ovaldb-url option. + +``` +$ vuls report -ovaldb-url=http://192.168.0.1:1323 +``` + +# Usage: Update OVAL Data + +- [RedHat, CentOS](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-redhat) +- [Ubuntu](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-ubuntu) +- [Debian](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-debian) +- [Oracle](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-oracle) ---- -# How to Update +# How to Update to the Latest Version - Update go-cve-dictionary -If the DB schema was changed, please specify new SQLite3 or MySQL DB file. +If the DB schema was changed, please specify new SQLite3, MySQL, PostgreSQL or Redis DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull @@ -1591,6 +1725,15 @@ $ rm -r vendor $ make install ``` +- Update goval-dictionary +If the DB schema was changed, please specify new SQLite3, MySQL, PostgreSQL or Redis DB file. +``` +$ cd $GOPATH/src/github.com/kotakanbe/goval-dictionary +$ git pull +$ rm -r vendor +$ make install +``` + - Update vuls ``` $ cd $GOPATH/src/github.com/future-architect/vuls @@ -1598,7 +1741,9 @@ $ git pull $ rm -r vendor $ make install ``` -Binary file was built under $GOPATH/bin + +- Binary file was built under $GOPATH/bin +- If an error occurs, delete `$GOPATH/pkg` before executing it --- @@ -1688,6 +1833,11 @@ kotakanbe ([@kotakanbe](https://twitter.com/kotakanbe)) created vuls and [these Please see [CHANGELOG](https://github.com/future-architect/vuls/blob/master/CHANGELOG.md). ---- +# Stargazers over time + +[![Stargazers over time](https://starcharts.herokuapp.com/future-architect/vuls.svg)](https://starcharts.herokuapp.com/future-architect/vuls) + +----- # License diff --git a/cache/bolt.go b/cache/bolt.go index 9edaf3b4..b2803c0d 100644 --- a/cache/bolt.go +++ b/cache/bolt.go @@ -22,9 +22,9 @@ import ( "fmt" "time" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/future-architect/vuls/util" + "github.com/sirupsen/logrus" ) // Bolt holds a pointer of bolt.DB diff --git a/cache/bolt_test.go b/cache/bolt_test.go index ea2a31b4..ed2bd38b 100644 --- a/cache/bolt_test.go +++ b/cache/bolt_test.go @@ -22,10 +22,10 @@ import ( "reflect" "testing" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" + "github.com/sirupsen/logrus" ) const path = "/tmp/vuls-test-cache-11111111.db" @@ -37,8 +37,8 @@ var meta = Meta{ Family: "ubuntu", Release: "16.04", }, - Packs: []models.PackageInfo{ - { + Packs: models.Packages{ + "apt": { Name: "apt", Version: "1", }, diff --git a/cache/db.go b/cache/db.go index 30299390..ce837774 100644 --- a/cache/db.go +++ b/cache/db.go @@ -45,16 +45,6 @@ type Cache interface { type Meta struct { Name string Distro config.Distro - Packs []models.PackageInfo + Packs models.Packages CreatedAt time.Time } - -// FindPack search a PackageInfo -func (m Meta) FindPack(name string) (pack models.PackageInfo, found bool) { - for _, p := range m.Packs { - if name == p.Name { - return p, true - } - } - return pack, false -} diff --git a/commands/cmdutil.go b/commands/cmdutil.go deleted file mode 100644 index 9a58019b..00000000 --- a/commands/cmdutil.go +++ /dev/null @@ -1,21 +0,0 @@ -package commands - -import ( - "fmt" - - "github.com/howeyc/gopass" -) - -func getPasswd(prompt string) (string, error) { - for { - fmt.Print(prompt) - pass, err := gopass.GetPasswdMasked() - if err != nil { - return "", fmt.Errorf("Failed to read password") - } - if 0 < len(pass) { - return string(pass[:]), nil - } - } - -} diff --git a/commands/configtest.go b/commands/configtest.go index 40809613..6feca8f5 100644 --- a/commands/configtest.go +++ b/commands/configtest.go @@ -36,6 +36,7 @@ type ConfigtestCmd struct { logDir string askKeyPassword bool containersOnly bool + deep bool sshNative bool httpProxy string timeoutSec int @@ -53,6 +54,7 @@ func (*ConfigtestCmd) Synopsis() string { return "Test configuration" } func (*ConfigtestCmd) Usage() string { return `configtest: configtest + [-deep] [-config=/path/to/config.toml] [-log-dir=/path/to/log] [-ask-key-password] @@ -86,6 +88,8 @@ func (p *ConfigtestCmd) SetFlags(f *flag.FlagSet) { "Ask ssh privatekey password before scanning", ) + f.BoolVar(&p.deep, "deep", false, "Config test for deep scan mode") + f.StringVar( &p.httpProxy, "http-proxy", @@ -133,6 +137,7 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa c.Conf.SSHNative = p.sshNative c.Conf.HTTPProxy = p.httpProxy c.Conf.ContainersOnly = p.containersOnly + c.Conf.Deep = p.deep var servernames []string if 0 < len(f.Args()) { @@ -169,7 +174,7 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa return subcommands.ExitFailure } - util.Log.Info("Checking dependendies...") + util.Log.Info("Checking dependencies...") scan.CheckDependencies(p.timeoutSec) util.Log.Info("Checking sudo settings...") diff --git a/commands/discover.go b/commands/discover.go index 36aeb77e..6f69ed26 100644 --- a/commands/discover.go +++ b/commands/discover.go @@ -27,8 +27,8 @@ import ( "github.com/google/subcommands" - "github.com/Sirupsen/logrus" ps "github.com/kotakanbe/go-pingscanner" + "github.com/sirupsen/logrus" ) // DiscoverCmd is Subcommand of host discovery mode @@ -87,9 +87,9 @@ func (p *DiscoverCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface return subcommands.ExitSuccess } -// Output the tmeplate of config.toml +// Output the template of config.toml func printConfigToml(ips []string) (err error) { - const tomlTempale = ` + const tomlTemplate = ` [slack] hookURL = "https://hooks.slack.com/services/abc123/defghijklmnopqrstuvwxyz" channel = "#channel-name" @@ -99,13 +99,13 @@ authUser = "username" notifyUsers = ["@username"] [email] -smtpAddr = "smtp.gmail.com" +smtpAddr = "smtp.example.com" smtpPort = "587" user = "username" password = "password" -from = "from@address.com" -to = ["to@address.com"] -cc = ["cc@address.com"] +from = "from@example.com" +to = ["to@example.com"] +cc = ["cc@example.com"] subjectPrefix = "[vuls]" [default] @@ -140,7 +140,7 @@ host = "{{$ip}}" # ["key", "value"], #] #[servers.{{index $names $i}}.containers] -#type = "docker" #or "lxd" defualt: docker +#type = "docker" #or "lxd" default: docker #includes = ["${running}"] #excludes = ["container_name_a", "4aa37a8b63b9"] @@ -149,7 +149,7 @@ host = "{{$ip}}" ` var tpl *template.Template - if tpl, err = template.New("tempalte").Parse(tomlTempale); err != nil { + if tpl, err = template.New("template").Parse(tomlTemplate); err != nil { return } @@ -167,7 +167,7 @@ host = "{{$ip}}" } a.Names = names - fmt.Println("# Create config.toml using below and then ./vuls --config=/path/to/config.toml") + fmt.Println("# Create config.toml using below and then ./vuls -config=/path/to/config.toml") if err = tpl.Execute(os.Stdout, a); err != nil { return } diff --git a/commands/history.go b/commands/history.go index 1c6acf4b..2a6e02fb 100644 --- a/commands/history.go +++ b/commands/history.go @@ -27,6 +27,7 @@ import ( "strings" c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/report" "github.com/google/subcommands" ) @@ -68,9 +69,8 @@ func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ c.Conf.DebugSQL = p.debugSQL c.Conf.ResultsDir = p.resultsDir - var err error - var dirs jsonDirs - if dirs, err = lsValidJSONDirs(); err != nil { + dirs, err := report.ListValidJSONDirs() + if err != nil { return subcommands.ExitFailure } for _, d := range dirs { diff --git a/commands/report.go b/commands/report.go index b1797742..1542347d 100644 --- a/commands/report.go +++ b/commands/report.go @@ -25,8 +25,8 @@ import ( "path/filepath" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/cveapi" "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/oval" "github.com/future-architect/vuls/report" "github.com/future-architect/vuls/util" "github.com/google/subcommands" @@ -46,9 +46,13 @@ type ReportCmd struct { ignoreUnscoredCves bool httpProxy string - cvedbtype string - cvedbpath string - cvedbURL string + cveDBType string + cveDBPath string + cveDBURL string + + ovalDBType string + ovalDBPath string + ovalDBURL string toSlack bool toEMail bool @@ -65,9 +69,10 @@ type ReportCmd struct { gzip bool - awsProfile string - awsS3Bucket string - awsRegion string + awsProfile string + awsS3Bucket string + awsS3ResultsDir string + awsRegion string azureAccount string azureKey string @@ -96,6 +101,9 @@ func (*ReportCmd) Usage() string { [-cvedb-type=sqlite3|mysql|postgres] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 or DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-cvss-over=7] [-diff] [-ignore-unscored-cves] @@ -114,7 +122,8 @@ func (*ReportCmd) Usage() string { [-aws-profile=default] [-aws-region=us-west-2] [-aws-s3-bucket=bucket_name] - [-azure-account=accout] + [-aws-s3-results-dir=/bucket/path/to/results] + [-azure-account=account] [-azure-key=key] [-azure-container=container] [-http-proxy=http://192.168.0.1:8080] @@ -150,23 +159,42 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) { "Refresh CVE information in JSON file under results dir") f.StringVar( - &p.cvedbtype, + &p.cveDBType, "cvedb-type", "sqlite3", "DB type for fetching CVE dictionary (sqlite3, mysql or postgres)") defaultCveDBPath := filepath.Join(wd, "cve.sqlite3") f.StringVar( - &p.cvedbpath, + &p.cveDBPath, "cvedb-path", defaultCveDBPath, "/path/to/sqlite3 (For get cve detail from cve.sqlite3)") f.StringVar( - &p.cvedbURL, + &p.cveDBURL, "cvedb-url", "", - "http://cve-dictionary.com:8080 or DB connection string") + "http://cve-dictionary.com:1323 or mysql connection string") + + f.StringVar( + &p.ovalDBType, + "ovaldb-type", + "sqlite3", + "DB type for fetching OVAL dictionary (sqlite3 or mysql)") + + defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3") + f.StringVar( + &p.ovalDBPath, + "ovaldb-path", + defaultOvalDBPath, + "/path/to/sqlite3 (For get oval detail from oval.sqlite3)") + + f.StringVar( + &p.ovalDBURL, + "ovaldb-url", + "", + "http://goval-dictionary.com:1324 or mysql connection string") f.Float64Var( &p.cvssScoreOver, @@ -237,6 +265,7 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS profile to use") f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS region to use") f.StringVar(&p.awsS3Bucket, "aws-s3-bucket", "", "S3 bucket name") + f.StringVar(&p.awsS3ResultsDir, "aws-s3-results-dir", "", "/bucket/path/to/results") f.BoolVar(&p.toAzureBlob, "to-azure-blob", @@ -273,15 +302,18 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.Lang = p.lang c.Conf.ResultsDir = p.resultsDir - c.Conf.CveDBType = p.cvedbtype - c.Conf.CveDBPath = p.cvedbpath - c.Conf.CveDBURL = p.cvedbURL + c.Conf.RefreshCve = p.refreshCve + c.Conf.Diff = p.diff + c.Conf.CveDBType = p.cveDBType + c.Conf.CveDBPath = p.cveDBPath + c.Conf.CveDBURL = p.cveDBURL + c.Conf.OvalDBType = p.ovalDBType + c.Conf.OvalDBPath = p.ovalDBPath + c.Conf.OvalDBURL = p.ovalDBURL c.Conf.CvssScoreOver = p.cvssScoreOver c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves c.Conf.HTTPProxy = p.httpProxy - c.Conf.Pipe = p.pipe - c.Conf.FormatXML = p.formatXML c.Conf.FormatJSON = p.formatJSON c.Conf.FormatOneEMail = p.formatOneEMail @@ -291,13 +323,14 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.GZIP = p.gzip c.Conf.Diff = p.diff + c.Conf.Pipe = p.pipe var dir string var err error if p.diff { - dir, err = jsonDir([]string{}) + dir, err = report.JSONDir([]string{}) } else { - dir, err = jsonDir(f.Args()) + dir, err = report.JSONDir(f.Args()) } if err != nil { util.Log.Errorf("Failed to read from JSON: %s", err) @@ -327,6 +360,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.AwsRegion = p.awsRegion c.Conf.AwsProfile = p.awsProfile c.Conf.S3Bucket = p.awsS3Bucket + c.Conf.S3ResultsDir = p.awsS3ResultsDir if err := report.CheckIfBucketExists(); err != nil { util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %s", c.Conf.S3Bucket, err) return subcommands.ExitUsageError @@ -347,7 +381,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.AzureContainer = p.azureContainer if len(c.Conf.AzureContainer) == 0 { - util.Log.Error("Azure storage container name is requied with --azure-container option") + util.Log.Error("Azure storage container name is required with -azure-container option") return subcommands.ExitUsageError } if err := report.CheckIfAzureContainerExists(); err != nil { @@ -366,9 +400,9 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} if !c.Conf.ValidateOnReport() { return subcommands.ExitUsageError } - if ok, err := cveapi.CveClient.CheckHealth(); !ok { + if err := report.CveClient.CheckHealth(); err != nil { util.Log.Errorf("CVE HTTP server is not running. err: %s", err) - util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with --cvedb-path option") + util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with -cvedb-path option") return subcommands.ExitFailure } if c.Conf.CveDBURL != "" { @@ -379,66 +413,25 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } } - var history models.ScanHistory - history, err = loadOneScanHistory(dir) - if err != nil { + if c.Conf.OvalDBURL != "" { + err := oval.Base{}.CheckHTTPHealth() + if err != nil { + util.Log.Errorf("OVAL HTTP server is not running. err: %s", err) + util.Log.Errorf("Run goval-dictionary as server mode before reporting or run with -ovaldb-path option") + return subcommands.ExitFailure + } + } + + var res models.ScanResults + if res, err = report.LoadScanResults(dir); err != nil { util.Log.Error(err) return subcommands.ExitFailure } util.Log.Infof("Loaded: %s", dir) - var results []models.ScanResult - for _, r := range history.ScanResults { - if p.refreshCve || needToRefreshCve(r) { - util.Log.Debugf("need to refresh") - if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" { - if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { - util.Log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", - c.Conf.CveDBPath) - return subcommands.ExitFailure - } - } - - filled, err := fillCveInfoFromCveDB(r) - if err != nil { - util.Log.Errorf("Failed to fill CVE information: %s", err) - return subcommands.ExitFailure - } - filled.Lang = c.Conf.Lang - if err := overwriteJSONFile(dir, *filled); err != nil { - util.Log.Errorf("Failed to write JSON: %s", err) - return subcommands.ExitFailure - } - results = append(results, *filled) - } else { - util.Log.Debugf("no need to refresh") - results = append(results, r) - } - } - - if p.diff { - currentHistory := models.ScanHistory{ScanResults: results} - previousHistory, err := loadPreviousScanHistory(currentHistory) - if err != nil { - util.Log.Error(err) - return subcommands.ExitFailure - } - - history, err = diff(currentHistory, previousHistory) - if err != nil { - util.Log.Error(err) - return subcommands.ExitFailure - } - results = []models.ScanResult{} - for _, r := range history.ScanResults { - filled, _ := r.FillCveDetail() - results = append(results, *filled) - } - } - - var res models.ScanResults - for _, r := range results { - res = append(res, r.FilterByCvssOver()) + if res, err = report.FillCveInfos(res, dir); err != nil { + util.Log.Error(err) + return subcommands.ExitFailure } for _, w := range reports { diff --git a/commands/scan.go b/commands/scan.go index 74128cb0..0ee4dae2 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -43,6 +43,7 @@ type ScanCmd struct { httpProxy string askKeyPassword bool containersOnly bool + deep bool skipBroken bool sshNative bool pipe bool @@ -60,6 +61,7 @@ func (*ScanCmd) Synopsis() string { return "Scan vulnerabilities" } func (*ScanCmd) Usage() string { return `scan: scan + [-deep] [-config=/path/to/config.toml] [-results-dir=/path/to/results] [-log-dir=/path/to/log] @@ -132,6 +134,12 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { "Ask ssh privatekey password before scanning", ) + f.BoolVar( + &p.deep, + "deep", + false, + "Deep scan mode. Scan accuracy improves and scanned information becomes richer. Since analysis of changelog, issue commands requiring sudo, but it may be slower and high load on the target server") + f.BoolVar( &p.pipe, "pipe", @@ -149,7 +157,7 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { &p.scanTimeoutSec, "timeout-scan", 120*60, - "Number of seconds for scaning vulnerabilities for all servers", + "Number of seconds for scanning vulnerabilities for all servers", ) } @@ -223,6 +231,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) c.Conf.SSHNative = p.sshNative c.Conf.HTTPProxy = p.httpProxy c.Conf.ContainersOnly = p.containersOnly + c.Conf.Deep = p.deep c.Conf.SkipBroken = p.skipBroken util.Log.Info("Validating config...") diff --git a/commands/tui.go b/commands/tui.go index 94347c6a..2e650a7e 100644 --- a/commands/tui.go +++ b/commands/tui.go @@ -38,12 +38,17 @@ type TuiCmd struct { configPath string logDir string - resultsDir string - refreshCve bool + resultsDir string + refreshCve bool + cvedbtype string cvedbpath string cveDictionaryURL string + ovalDBType string + ovalDBPath string + ovalDBURL string + pipe bool } @@ -51,7 +56,7 @@ type TuiCmd struct { func (*TuiCmd) Name() string { return "tui" } // Synopsis return synopsis -func (*TuiCmd) Synopsis() string { return "Run Tui view to anayze vulnerabilites" } +func (*TuiCmd) Synopsis() string { return "Run Tui view to analyze vulnerabilities" } // Usage return usage func (*TuiCmd) Usage() string { @@ -61,6 +66,9 @@ func (*TuiCmd) Usage() string { [-cvedb-type=sqlite3|mysql|postgres] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 or DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-refresh-cve] [-results-dir=/path/to/results] [-log-dir=/path/to/log] @@ -110,7 +118,26 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) { &p.cveDictionaryURL, "cvedb-url", "", - "http://cve-dictionary.com:8080 or DB connection string") + "http://cve-dictionary.example.com:1323 or mysql connection string") + + f.StringVar( + &p.ovalDBType, + "ovaldb-type", + "sqlite3", + "DB type for fetching OVAL dictionary (sqlite3 or mysql)") + + defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3") + f.StringVar( + &p.ovalDBPath, + "ovaldb-path", + defaultOvalDBPath, + "/path/to/sqlite3 (For get oval detail from oval.sqlite3)") + + f.StringVar( + &p.ovalDBURL, + "ovaldb-url", + "", + "http://goval-dictionary.example.com:1324 or mysql connection string") f.BoolVar( &p.pipe, @@ -139,6 +166,9 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s c.Conf.CveDBType = p.cvedbtype c.Conf.CveDBPath = p.cvedbpath c.Conf.CveDBURL = p.cveDictionaryURL + c.Conf.OvalDBType = p.ovalDBType + c.Conf.OvalDBPath = p.ovalDBPath + c.Conf.OvalDBURL = p.ovalDBURL log.Info("Validating config...") if !c.Conf.ValidateOnTui() { @@ -146,44 +176,22 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } c.Conf.Pipe = p.pipe - jsonDir, err := jsonDir(f.Args()) + + dir, err := report.JSONDir(f.Args()) if err != nil { - log.Errorf("Failed to read json dir under results: %s", err) + util.Log.Errorf("Failed to read from JSON: %s", err) return subcommands.ExitFailure } - - history, err := loadOneScanHistory(jsonDir) - if err != nil { - log.Errorf("Failed to read from JSON: %s", err) + var res models.ScanResults + if res, err = report.LoadScanResults(dir); err != nil { + util.Log.Error(err) return subcommands.ExitFailure } + util.Log.Infof("Loaded: %s", dir) - var results []models.ScanResult - for _, r := range history.ScanResults { - if p.refreshCve || needToRefreshCve(r) { - if c.Conf.CveDBType == "sqlite3" { - if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { - log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", - c.Conf.CveDBPath) - return subcommands.ExitFailure - } - } - - filled, err := fillCveInfoFromCveDB(r) - if err != nil { - log.Errorf("Failed to fill CVE information: %s", err) - return subcommands.ExitFailure - } - - if err := overwriteJSONFile(jsonDir, *filled); err != nil { - log.Errorf("Failed to write JSON: %s", err) - return subcommands.ExitFailure - } - results = append(results, *filled) - } else { - results = append(results, r) - } + if res, err = report.FillCveInfos(res, dir); err != nil { + util.Log.Error(err) + return subcommands.ExitFailure } - history.ScanResults = results - return report.RunTui(history) + return report.RunTui(res) } diff --git a/commands/util.go b/commands/util.go index 73a44ad3..d0a08b5f 100644 --- a/commands/util.go +++ b/commands/util.go @@ -18,316 +18,21 @@ along with this program. If not, see . package commands import ( - "encoding/json" "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "time" - c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/cveapi" - "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" - "github.com/future-architect/vuls/util" + "github.com/howeyc/gopass" ) -// jsonDirPattern is file name pattern of JSON directory -// 2016-11-16T10:43:28+09:00 -// 2016-11-16T10:43:28Z -var jsonDirPattern = regexp.MustCompile( - `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$`) - -// JSONDirs is array of json files path. -type jsonDirs []string - -// sort as recent directories are at the head -func (d jsonDirs) Len() int { - return len(d) -} -func (d jsonDirs) Swap(i, j int) { - d[i], d[j] = d[j], d[i] -} -func (d jsonDirs) Less(i, j int) bool { - return d[j] < d[i] -} - -// getValidJSONDirs return valid json directory as array -// Returned array is sorted so that recent directories are at the head -func lsValidJSONDirs() (dirs jsonDirs, err error) { - var dirInfo []os.FileInfo - if dirInfo, err = ioutil.ReadDir(c.Conf.ResultsDir); err != nil { - err = fmt.Errorf("Failed to read %s: %s", c.Conf.ResultsDir, err) - return - } - for _, d := range dirInfo { - if d.IsDir() && jsonDirPattern.MatchString(d.Name()) { - jsonDir := filepath.Join(c.Conf.ResultsDir, d.Name()) - dirs = append(dirs, jsonDir) - } - } - sort.Sort(dirs) - return -} - -// jsonDir returns -// If there is an arg, check if it is a valid format and return the corresponding path under results. -// If arg passed via PIPE (such as history subcommand), return that path. -// Otherwise, returns the path of the latest directory -func jsonDir(args []string) (string, error) { - var err error - var dirs jsonDirs - - if 0 < len(args) { - if dirs, err = lsValidJSONDirs(); err != nil { - return "", err - } - - path := filepath.Join(c.Conf.ResultsDir, args[0]) - for _, d := range dirs { - ss := strings.Split(d, string(os.PathSeparator)) - timedir := ss[len(ss)-1] - if timedir == args[0] { - return path, nil - } - } - - return "", fmt.Errorf("Invalid path: %s", path) - } - - // PIPE - if c.Conf.Pipe { - bytes, err := ioutil.ReadAll(os.Stdin) +func getPasswd(prompt string) (string, error) { + for { + fmt.Print(prompt) + pass, err := gopass.GetPasswdMasked() if err != nil { - return "", fmt.Errorf("Failed to read stdin: %s", err) + return "", fmt.Errorf("Failed to read password") } - fields := strings.Fields(string(bytes)) - if 0 < len(fields) { - return filepath.Join(c.Conf.ResultsDir, fields[0]), nil + if 0 < len(pass) { + return string(pass[:]), nil } - return "", fmt.Errorf("Stdin is invalid: %s", string(bytes)) } - // returns latest dir when no args or no PIPE - if dirs, err = lsValidJSONDirs(); err != nil { - return "", err - } - if len(dirs) == 0 { - return "", fmt.Errorf("No results under %s", - c.Conf.ResultsDir) - } - return dirs[0], nil -} - -// loadOneServerScanResult read JSON data of one server -func loadOneServerScanResult(jsonFile string) (result models.ScanResult, err error) { - var data []byte - if data, err = ioutil.ReadFile(jsonFile); err != nil { - err = fmt.Errorf("Failed to read %s: %s", jsonFile, err) - return - } - if json.Unmarshal(data, &result) != nil { - err = fmt.Errorf("Failed to parse %s: %s", jsonFile, err) - } - return -} - -// loadOneScanHistory read JSON data -func loadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err error) { - var results []models.ScanResult - var files []os.FileInfo - if files, err = ioutil.ReadDir(jsonDir); err != nil { - err = fmt.Errorf("Failed to read %s: %s", jsonDir, err) - return - } - for _, f := range files { - if filepath.Ext(f.Name()) != ".json" || strings.HasSuffix(f.Name(), "_diff.json") { - continue - } - - var r models.ScanResult - path := filepath.Join(jsonDir, f.Name()) - if r, err = loadOneServerScanResult(path); err != nil { - return - } - - results = append(results, r) - } - if len(results) == 0 { - err = fmt.Errorf("There is no json file under %s", jsonDir) - return - } - - scanHistory = models.ScanHistory{ - ScanResults: results, - } - return -} - -func fillCveInfoFromCveDB(r models.ScanResult) (*models.ScanResult, error) { - var err error - var vs []models.VulnInfo - - sInfo := c.Conf.Servers[r.ServerName] - vs, err = scanVulnByCpeNames(sInfo.CpeNames, r.ScannedCves) - if err != nil { - return nil, err - } - r.ScannedCves = vs - return r.FillCveDetail() -} - -func loadPreviousScanHistory(current models.ScanHistory) (previous models.ScanHistory, err error) { - var dirs jsonDirs - if dirs, err = lsValidJSONDirs(); err != nil { - return - } - - for _, result := range current.ScanResults { - for _, dir := range dirs[1:] { - var r models.ScanResult - path := filepath.Join(dir, result.ServerName+".json") - if r, err = loadOneServerScanResult(path); err != nil { - continue - } - if r.Family == result.Family && r.Release == result.Release { - previous.ScanResults = append(previous.ScanResults, r) - break - } - } - } - return previous, nil -} - -func diff(currentHistory, previousHistory models.ScanHistory) (diffHistory models.ScanHistory, err error) { - for _, currentResult := range currentHistory.ScanResults { - found := false - var previousResult models.ScanResult - for _, previousResult = range previousHistory.ScanResults { - if currentResult.ServerName == previousResult.ServerName { - found = true - break - } - } - - if found { - currentResult.ScannedCves = getNewCves(previousResult, currentResult) - - currentResult.KnownCves = []models.CveInfo{} - currentResult.UnknownCves = []models.CveInfo{} - - currentResult.Packages = models.PackageInfoList{} - for _, s := range currentResult.ScannedCves { - currentResult.Packages = append(currentResult.Packages, s.Packages...) - } - currentResult.Packages = currentResult.Packages.UniqByName() - } - - diffHistory.ScanResults = append(diffHistory.ScanResults, currentResult) - } - return diffHistory, err -} - -func getNewCves(previousResult, currentResult models.ScanResult) (newVulninfos []models.VulnInfo) { - previousCveIDsSet := map[string]bool{} - for _, previousVulnInfo := range previousResult.ScannedCves { - previousCveIDsSet[previousVulnInfo.CveID] = true - } - - for _, v := range currentResult.ScannedCves { - if previousCveIDsSet[v.CveID] { - if isCveInfoUpdated(currentResult, previousResult, v.CveID) { - newVulninfos = append(newVulninfos, v) - } - } else { - newVulninfos = append(newVulninfos, v) - } - } - return -} - -func isCveInfoUpdated(currentResult, previousResult models.ScanResult, CveID string) bool { - type lastModified struct { - Nvd time.Time - Jvn time.Time - } - - previousModifies := lastModified{} - for _, c := range previousResult.KnownCves { - if CveID == c.CveID { - previousModifies.Nvd = c.CveDetail.Nvd.LastModifiedDate - previousModifies.Jvn = c.CveDetail.Jvn.LastModifiedDate - } - } - - currentModifies := lastModified{} - for _, c := range currentResult.KnownCves { - if CveID == c.CveDetail.CveID { - currentModifies.Nvd = c.CveDetail.Nvd.LastModifiedDate - currentModifies.Jvn = c.CveDetail.Jvn.LastModifiedDate - } - } - return !currentModifies.Nvd.Equal(previousModifies.Nvd) || - !currentModifies.Jvn.Equal(previousModifies.Jvn) -} - -func overwriteJSONFile(dir string, r models.ScanResult) error { - before := c.Conf.FormatJSON - beforeDiff := c.Conf.Diff - c.Conf.FormatJSON = true - c.Conf.Diff = false - w := report.LocalFileWriter{CurrentDir: dir} - if err := w.Write(r); err != nil { - return fmt.Errorf("Failed to write summary report: %s", err) - } - c.Conf.FormatJSON = before - c.Conf.Diff = beforeDiff - return nil -} - -func scanVulnByCpeNames(cpeNames []string, scannedVulns []models.VulnInfo) ([]models.VulnInfo, error) { - // To remove duplicate - set := map[string]models.VulnInfo{} - for _, v := range scannedVulns { - set[v.CveID] = v - } - - for _, name := range cpeNames { - details, err := cveapi.CveClient.FetchCveDetailsByCpeName(name) - if err != nil { - return nil, err - } - for _, detail := range details { - if val, ok := set[detail.CveID]; ok { - names := val.CpeNames - names = util.AppendIfMissing(names, name) - val.CpeNames = names - val.Confidence = models.CpeNameMatch - set[detail.CveID] = val - } else { - v := models.VulnInfo{ - CveID: detail.CveID, - CpeNames: []string{name}, - Confidence: models.CpeNameMatch, - } - v.NilSliceToEmpty() - set[detail.CveID] = v - } - } - } - - vinfos := []models.VulnInfo{} - for key := range set { - vinfos = append(vinfos, set[key]) - } - return vinfos, nil -} - -func needToRefreshCve(r models.ScanResult) bool { - return r.Lang != c.Conf.Lang || len(r.KnownCves) == 0 && - len(r.UnknownCves) == 0 && - len(r.IgnoredCves) == 0 } diff --git a/commands/util_test.go b/commands/util_test.go index ddcde598..e9ab19df 100644 --- a/commands/util_test.go +++ b/commands/util_test.go @@ -16,279 +16,3 @@ along with this program. If not, see . */ package commands - -import ( - "testing" - "time" - - "reflect" - - "github.com/future-architect/vuls/models" - "github.com/k0kubun/pp" - cve "github.com/kotakanbe/go-cve-dictionary/models" -) - -func TestDiff(t *testing.T) { - atCurrent, _ := time.Parse("2006-01-02", "2014-12-31") - atPrevious, _ := time.Parse("2006-01-02", "2014-11-31") - var tests = []struct { - inCurrent models.ScanHistory - inPrevious models.ScanHistory - out models.ScanResult - }{ - { - models.ScanHistory{ - ScanResults: models.ScanResults{ - { - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: []models.VulnInfo{ - { - CveID: "CVE-2012-6702", - Packages: models.PackageInfoList{ - { - Name: "libexpat1", - Version: "2.1.0-7", - Release: "", - NewVersion: "2.1.0-7ubuntu0.16.04.2", - NewRelease: "", - Repository: "", - }, - }, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - { - CveID: "CVE-2014-9761", - Packages: models.PackageInfoList{ - { - Name: "libc-bin", - Version: "2.21-0ubuntu5", - Release: "", - NewVersion: "2.23-0ubuntu5", - NewRelease: "", - Repository: "", - }, - }, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - KnownCves: []models.CveInfo{}, - UnknownCves: []models.CveInfo{}, - IgnoredCves: []models.CveInfo{}, - - Packages: models.PackageInfoList{}, - - Errors: []string{}, - Optional: [][]interface{}{}, - }, - }, - }, - models.ScanHistory{ - ScanResults: models.ScanResults{ - { - ScannedAt: atPrevious, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: []models.VulnInfo{ - { - CveID: "CVE-2012-6702", - Packages: models.PackageInfoList{ - { - Name: "libexpat1", - Version: "2.1.0-7", - Release: "", - NewVersion: "2.1.0-7ubuntu0.16.04.2", - NewRelease: "", - Repository: "", - }, - }, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - { - CveID: "CVE-2014-9761", - Packages: models.PackageInfoList{ - { - Name: "libc-bin", - Version: "2.21-0ubuntu5", - Release: "", - NewVersion: "2.23-0ubuntu5", - NewRelease: "", - Repository: "", - }, - }, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - KnownCves: []models.CveInfo{}, - UnknownCves: []models.CveInfo{}, - IgnoredCves: []models.CveInfo{}, - - Packages: models.PackageInfoList{}, - - Errors: []string{}, - Optional: [][]interface{}{}, - }, - }, - }, - models.ScanResult{ - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - KnownCves: []models.CveInfo{}, - UnknownCves: []models.CveInfo{}, - IgnoredCves: []models.CveInfo{}, - - // Packages: models.PackageInfoList{}, - - Errors: []string{}, - Optional: [][]interface{}{}, - }, - }, - { - models.ScanHistory{ - ScanResults: models.ScanResults{ - { - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: []models.VulnInfo{ - { - CveID: "CVE-2016-6662", - Packages: models.PackageInfoList{ - { - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - }, - }, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - KnownCves: []models.CveInfo{ - { - CveDetail: cve.CveDetail{ - CveID: "CVE-2016-6662", - Nvd: cve.Nvd{ - LastModifiedDate: time.Date(2016, 1, 1, 0, 0, 0, 0, time.Local), - }, - }, - VulnInfo: models.VulnInfo{ - CveID: "CVE-2016-6662", - }, - }, - }, - UnknownCves: []models.CveInfo{}, - IgnoredCves: []models.CveInfo{}, - }, - }, - }, - models.ScanHistory{ - ScanResults: models.ScanResults{ - { - ScannedAt: atPrevious, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: []models.VulnInfo{ - { - CveID: "CVE-2016-6662", - Packages: models.PackageInfoList{ - { - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - }, - }, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - KnownCves: []models.CveInfo{ - { - CveDetail: cve.CveDetail{ - CveID: "CVE-2016-6662", - Nvd: cve.Nvd{ - LastModifiedDate: time.Date(2017, 3, 15, 13, 40, 57, 0, time.Local), - }, - }, - VulnInfo: models.VulnInfo{ - CveID: "CVE-2016-6662", - }, - }, - }, - UnknownCves: []models.CveInfo{}, - IgnoredCves: []models.CveInfo{}, - }, - }, - }, - models.ScanResult{ - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: []models.VulnInfo{ - { - CveID: "CVE-2016-6662", - Packages: models.PackageInfoList{ - { - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - }, - }, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - KnownCves: []models.CveInfo{}, - UnknownCves: []models.CveInfo{}, - IgnoredCves: []models.CveInfo{}, - Packages: models.PackageInfoList{ - models.PackageInfo{ - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - Changelog: models.Changelog{ - Contents: "", - Method: "", - }, - }, - }, - }, - }, - } - - var s models.ScanHistory - for _, tt := range tests { - s, _ = diff(tt.inCurrent, tt.inPrevious) - for _, actual := range s.ScanResults { - if !reflect.DeepEqual(actual, tt.out) { - h := pp.Sprint(actual) - x := pp.Sprint(tt.out) - t.Errorf("diff result : \n %s \n output result : \n %s", h, x) - } - } - } -} diff --git a/config/config.go b/config/config.go index ff42bfd4..033ecc35 100644 --- a/config/config.go +++ b/config/config.go @@ -19,17 +19,47 @@ package config import ( "fmt" + "os" "runtime" "strconv" "strings" - log "github.com/Sirupsen/logrus" valid "github.com/asaskevich/govalidator" + log "github.com/sirupsen/logrus" ) // Conf has Configuration var Conf Config +const ( + // RedHat is + RedHat = "redhat" + + // Debian is + Debian = "debian" + + // Ubuntu is + Ubuntu = "ubuntu" + + // CentOS is + CentOS = "centos" + + // Fedora is + Fedora = "fedora" + + // Amazon is + Amazon = "amazon" + + // Oracle is + Oracle = "oracle" + + // FreeBSD is + FreeBSD = "freebsd" + + // Raspbian is + Raspbian = "raspbian" +) + //Config is struct of Configuration type Config struct { Debug bool @@ -46,17 +76,25 @@ type Config struct { SSHNative bool ContainersOnly bool + Deep bool SkipBroken bool HTTPProxy string `valid:"url"` LogDir string ResultsDir string - CveDBType string - CveDBPath string - CveDBURL string + CveDBType string + CveDBPath string + CveDBURL string + + OvalDBType string + OvalDBPath string + OvalDBURL string + CacheDBPath string + RefreshCve bool + FormatXML bool FormatJSON bool FormatOneEMail bool @@ -66,12 +104,13 @@ type Config struct { GZIP bool - AwsProfile string - AwsRegion string - S3Bucket string + AwsProfile string + AwsRegion string + S3Bucket string + S3ResultsDir string AzureAccount string - AzureKey string + AzureKey string `json:"-"` AzureContainer string Pipe bool @@ -155,26 +194,17 @@ func (c Config) ValidateOnReport() bool { } } - switch c.CveDBType { - case "sqlite3": - if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { - errs = append(errs, fmt.Errorf( - "SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cvedb-path: %s", - c.CveDBPath)) + if err := validateDB("cvedb", c.CveDBType, c.CveDBPath, c.CveDBURL); err != nil { + errs = append(errs, err) + } + if c.CveDBType == "sqlite3" { + if _, err := os.Stat(c.CveDBPath); os.IsNotExist(err) { + errs = append(errs, fmt.Errorf("SQLite3 DB path (%s) is not exist: %s", "cvedb", c.CveDBPath)) } - case "mysql": - if c.CveDBURL == "" { - errs = append(errs, fmt.Errorf( - `MySQL connection string is needed. -cvedb-url="user:pass@tcp(localhost:3306)/dbname"`)) - } - case "postgres": - if c.CveDBURL == "" { - errs = append(errs, fmt.Errorf( - `PostgreSQL connection string is needed. -cvedb-url=""host=myhost user=user dbname=dbname sslmode=disable password=password""`)) - } - default: - errs = append(errs, fmt.Errorf( - "CVE DB type must be either 'sqlite3', 'mysql' or 'postgres'. -cvedb-type: %s", c.CveDBType)) + } + + if err := validateDB("ovaldb", c.OvalDBType, c.OvalDBPath, c.OvalDBURL); err != nil { + errs = append(errs, err) } _, err := valid.ValidateStruct(c) @@ -208,15 +238,12 @@ func (c Config) ValidateOnTui() bool { } } - if c.CveDBType != "sqlite3" && c.CveDBType != "mysql" && c.CveDBType != "postgres" { - errs = append(errs, fmt.Errorf( - "CVE DB type must be either 'sqlite3', 'mysql' or 'postgres'. -cve-dictionary-dbtype: %s", c.CveDBType)) + if err := validateDB("cvedb", c.CveDBType, c.CveDBPath, c.CveDBURL); err != nil { + errs = append(errs, err) } - if c.CveDBType == "sqlite3" { - if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { - errs = append(errs, fmt.Errorf( - "SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath)) + if _, err := os.Stat(c.CveDBPath); os.IsNotExist(err) { + errs = append(errs, fmt.Errorf("SQLite3 DB path (%s) is not exist: %s", "cvedb", c.CveDBPath)) } } @@ -227,13 +254,53 @@ func (c Config) ValidateOnTui() bool { return len(errs) == 0 } +// validateDB validates configuration +// dictionaryDB name is 'cvedb' or 'ovaldb' +func validateDB(dictionaryDBName, dbType, dbPath, dbURL string) error { + switch dbType { + case "sqlite3": + if ok, _ := valid.IsFilePath(dbPath); !ok { + return fmt.Errorf( + "SQLite3 DB path (%s) must be a *Absolute* file path. -%s-path: %s", + dictionaryDBName, + dictionaryDBName, + dbPath) + } + case "mysql": + if dbURL == "" { + return fmt.Errorf( + `MySQL connection string is needed. -%s-url="user:pass@tcp(localhost:3306)/dbname"`, + dictionaryDBName) + } + case "postgres": + if dbURL == "" { + return fmt.Errorf( + `PostgreSQL connection string is needed. -%s-url="host=myhost user=user dbname=dbname sslmode=disable password=password"`, + dictionaryDBName) + } + case "redis": + if dbURL == "" { + return fmt.Errorf( + `Redis connection string is needed. -%s-url="redis://localhost/0"`, + dictionaryDBName) + } + default: + return fmt.Errorf( + "%s type must be either 'sqlite3', 'mysql', 'postgres' or 'redis'. -%s-type: %s", + dictionaryDBName, + dictionaryDBName, + dbType) + } + return nil +} + // SMTPConf is smtp config type SMTPConf struct { SMTPAddr string SMTPPort string `valid:"port"` User string - Password string + Password string `json:"-"` From string To []string Cc []string @@ -293,7 +360,7 @@ func (c *SMTPConf) Validate() (errs []error) { // SlackConf is slack config type SlackConf struct { - HookURL string `valid:"url"` + HookURL string `valid:"url" json:"-"` Channel string `json:"channel"` IconEmoji string `json:"icon_emoji"` AuthUser string `json:"username"` @@ -343,7 +410,7 @@ type ServerInfo struct { Host string Port string KeyPath string - KeyPassword string + KeyPassword string `json:"-"` CpeNames []string DependencyCheckXMLPath string @@ -357,7 +424,7 @@ type ServerInfo struct { Optional [][]interface{} // For CentOS, RHEL, Amazon - Enablerepo string + Enablerepo []string // used internal LogMsgAnsiColor string // DebugLog Color diff --git a/config/tomlloader.go b/config/tomlloader.go index 6ed77023..1cadd8cc 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -20,11 +20,10 @@ package config import ( "fmt" "os" - "strings" "github.com/BurntSushi/toml" - log "github.com/Sirupsen/logrus" "github.com/future-architect/vuls/contrib/owasp-dependency-check/parser" + log "github.com/sirupsen/logrus" ) // TOMLLoader loads config @@ -164,7 +163,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { s.Enablerepo = d.Enablerepo } if len(s.Enablerepo) != 0 { - for _, repo := range strings.Split(s.Enablerepo, ",") { + for _, repo := range s.Enablerepo { switch repo { case "base", "updates": // nop diff --git a/contrib/owasp-dependency-check/parser/parser.go b/contrib/owasp-dependency-check/parser/parser.go index c0e8e818..90857a09 100644 --- a/contrib/owasp-dependency-check/parser/parser.go +++ b/contrib/owasp-dependency-check/parser/parser.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "os" - "sort" "strings" ) @@ -35,18 +34,18 @@ func appendIfMissing(slice []string, str string) []string { func Parse(path string) ([]string, error) { file, err := os.Open(path) if err != nil { - return []string{}, fmt.Errorf("Failed to open: %s", err) + return nil, fmt.Errorf("Failed to open: %s", err) } defer file.Close() b, err := ioutil.ReadAll(file) if err != nil { - return []string{}, fmt.Errorf("Failed to read: %s", err) + return nil, fmt.Errorf("Failed to read: %s", err) } var anal analysis if err := xml.Unmarshal(b, &anal); err != nil { - fmt.Errorf("Failed to unmarshal: %s", err) + return nil, fmt.Errorf("Failed to unmarshal: %s", err) } cpes := []string{} @@ -59,6 +58,5 @@ func Parse(path string) ([]string, error) { } } } - sort.Strings(cpes) return cpes, nil } diff --git a/img/vuls-abstract.png b/img/vuls-abstract.png new file mode 100644 index 00000000..291d87ad Binary files /dev/null and b/img/vuls-abstract.png differ diff --git a/img/vuls-architecture-localscan.graphml b/img/vuls-architecture-localscan.graphml index 665bd2af..11b4e0d9 100644 --- a/img/vuls-architecture-localscan.graphml +++ b/img/vuls-architecture-localscan.graphml @@ -1,6 +1,6 @@ - + @@ -17,10 +17,10 @@ - + - + @@ -37,20 +37,20 @@ - + - Vulnerbility Database + Vulnerbility Database - + - Folder 1 + Folder 1 @@ -63,10 +63,10 @@ - + - JVN + JVN (Japanese) @@ -81,10 +81,27 @@ - + - NVD + NVD + + + + + + + + + + + + + + + + + OVAL @@ -106,17 +123,17 @@ - Distribution Support + Distribution Support - + - Folder 2 + Folder 2 @@ -132,7 +149,7 @@ - apptitude + apptitude changelog @@ -150,7 +167,7 @@ changelog - yum + yum changelog @@ -165,11 +182,12 @@ changelog - + - RHSA (RedHat) -ALAS (Amazon) + RHSA (RedHat) +ALAS (Amazon) +ELSA(Oracle) @@ -186,7 +204,7 @@ ALAS (Amazon) - FreeBSD Support + FreeBSD Support @@ -205,7 +223,7 @@ ALAS (Amazon) - + @@ -222,7 +240,7 @@ ALAS (Amazon) - System Operator + System Operator @@ -242,7 +260,7 @@ ALAS (Amazon) - + @@ -259,7 +277,7 @@ ALAS (Amazon) - go-cve-dictionary + go-cve-dictionary / goval-dictionary @@ -269,7 +287,7 @@ ALAS (Amazon) - Folder 4 + Folder 4 @@ -285,7 +303,7 @@ ALAS (Amazon) - + @@ -293,7 +311,7 @@ ALAS (Amazon) - SQLite3 + SQLite3 @@ -309,7 +327,7 @@ ALAS (Amazon) - HTTP server + HTTP server @@ -326,7 +344,7 @@ ALAS (Amazon) - Fetcher + Fetcher @@ -345,7 +363,7 @@ ALAS (Amazon) - + @@ -356,7 +374,7 @@ ALAS (Amazon) - Azure + Azure BLOB @@ -374,7 +392,7 @@ BLOB - .xml + .xml @@ -390,7 +408,7 @@ BLOB - .txt + .txt @@ -406,7 +424,7 @@ BLOB - .json + .json @@ -422,7 +440,7 @@ BLOB - .gz + .gz @@ -435,10 +453,10 @@ BLOB - + - + @@ -464,7 +482,7 @@ BLOB - Vuls Reporting Server + Vuls Reporting Server @@ -474,7 +492,7 @@ BLOB - Folder 10 + Folder 10 @@ -493,7 +511,7 @@ BLOB - Vuls + Vuls @@ -503,7 +521,7 @@ BLOB - Folder 9 + Folder 9 @@ -519,7 +537,7 @@ BLOB - Report + Report @@ -536,7 +554,7 @@ BLOB - VulsRepo + VulsRepo (WebUI) @@ -554,7 +572,7 @@ BLOB - TUI + TUI @@ -574,7 +592,7 @@ BLOB - results dir + results dir @@ -584,7 +602,7 @@ BLOB - Folder 7 + Folder 7 @@ -600,7 +618,7 @@ BLOB - JSON + JSON @@ -617,7 +635,7 @@ BLOB - JSON + JSON @@ -634,7 +652,7 @@ BLOB - JSON + JSON @@ -660,7 +678,7 @@ BLOB - Scan Target Server + Scan Target Server @@ -670,7 +688,7 @@ BLOB - Folder 10 + Folder 10 @@ -689,7 +707,7 @@ BLOB - Docker/LXD + Docker/LXD @@ -699,7 +717,7 @@ BLOB - Folder 5 + Folder 5 @@ -715,7 +733,7 @@ BLOB - Container + Container @@ -734,7 +752,7 @@ BLOB - Package Manager + Package Manager @@ -754,7 +772,7 @@ BLOB - Vuls + Vuls @@ -764,7 +782,7 @@ BLOB - Folder 10 + Folder 10 @@ -780,7 +798,7 @@ BLOB - Scan + Scan @@ -800,7 +818,7 @@ BLOB - results dir + results dir @@ -810,7 +828,7 @@ BLOB - Folder 7 + Folder 7 @@ -826,7 +844,7 @@ BLOB - JSON + JSON @@ -843,7 +861,7 @@ BLOB - JSON + JSON @@ -860,7 +878,7 @@ BLOB - JSON + JSON @@ -877,32 +895,13 @@ BLOB - - - - - - - Fetch -Vulnerability data - - - - - - - - - - - - + - HTTP + HTTP @@ -914,38 +913,12 @@ Vulnerability data - - - - - - - HTTP - - - - - - - - - - - - + - WebUI - - - - - - - @@ -956,7 +929,7 @@ Vulnerability data - docker exec + docker exec lxc exec @@ -969,7 +942,7 @@ lxc exec - + @@ -985,7 +958,7 @@ lxc exec - Insert + Insert @@ -997,31 +970,13 @@ lxc exec - - - - - - - Notify - - - - - - - - - - - - Select + Select @@ -1043,7 +998,7 @@ lxc exec - + @@ -1053,7 +1008,7 @@ lxc exec - + @@ -1073,13 +1028,13 @@ lxc exec - + - Put + Put @@ -1091,7 +1046,7 @@ lxc exec - + @@ -1111,13 +1066,13 @@ lxc exec - + - View Results + View Results on Terminal @@ -1136,7 +1091,7 @@ lxc exec - os.exec + os.exec @@ -1148,13 +1103,13 @@ lxc exec - + - HTTP + HTTP @@ -1176,13 +1131,13 @@ lxc exec - + - transfer + transfer @@ -1194,7 +1149,17 @@ lxc exec - + + + + + + + + + + + diff --git a/img/vuls-architecture-localscan.png b/img/vuls-architecture-localscan.png index bd48bdcf..e703d121 100644 Binary files a/img/vuls-architecture-localscan.png and b/img/vuls-architecture-localscan.png differ diff --git a/img/vuls-architecture.graphml b/img/vuls-architecture.graphml index b29d958c..9e263988 100644 --- a/img/vuls-architecture.graphml +++ b/img/vuls-architecture.graphml @@ -1,6 +1,6 @@ - + @@ -20,7 +20,7 @@ - + @@ -37,20 +37,20 @@ - + - Vulnerbility Database + Vulnerbility Database - + - Folder 1 + Folder 1 @@ -63,10 +63,10 @@ - + - JVN + JVN (Japanese) @@ -81,10 +81,27 @@ - + - NVD + NVD + + + + + + + + + + + + + + + + + OVAL @@ -103,20 +120,20 @@ - + - Distribution Support + Distribution Support - + - Folder 2 + Folder 2 @@ -132,7 +149,7 @@ - apptitude + apptitude changelog @@ -150,7 +167,7 @@ changelog - yum + yum changelog @@ -165,11 +182,12 @@ changelog - + - RHSA (RedHat) -ALAS (Amazon) + RHSA (RedHat) +ALAS (Amazon) +ELSA (Oracle) @@ -183,10 +201,10 @@ ALAS (Amazon) - + - FreeBSD Support + FreeBSD Support @@ -205,7 +223,7 @@ ALAS (Amazon) - + @@ -225,7 +243,7 @@ ALAS (Amazon) - Vuls + Vuls @@ -235,7 +253,7 @@ ALAS (Amazon) - Folder 3 + Folder 3 @@ -251,7 +269,7 @@ ALAS (Amazon) - Scan + Scan @@ -268,7 +286,7 @@ ALAS (Amazon) - Report + Report @@ -285,7 +303,7 @@ ALAS (Amazon) - VulsRepo + VulsRepo (WebUI) @@ -303,7 +321,7 @@ ALAS (Amazon) - TUI + TUI @@ -322,7 +340,7 @@ ALAS (Amazon) - System Operator + System Operator @@ -342,7 +360,7 @@ ALAS (Amazon) - + @@ -356,7 +374,7 @@ ALAS (Amazon) - + @@ -382,7 +400,7 @@ ALAS (Amazon) - go-cve-dictionary + go-cve-dictionary / goval-dictionary @@ -392,7 +410,7 @@ ALAS (Amazon) - Folder 4 + Folder 4 @@ -408,7 +426,7 @@ ALAS (Amazon) - + @@ -416,7 +434,7 @@ ALAS (Amazon) - SQLite3 + SQLite3 @@ -432,7 +450,7 @@ ALAS (Amazon) - HTTP server + HTTP server @@ -449,7 +467,7 @@ ALAS (Amazon) - Fetcher + Fetcher @@ -471,7 +489,7 @@ ALAS (Amazon) - Docker/LXD + Docker/LXD @@ -481,7 +499,7 @@ ALAS (Amazon) - Folder 5 + Folder 5 @@ -497,7 +515,7 @@ ALAS (Amazon) - Host + Host @@ -514,7 +532,7 @@ ALAS (Amazon) - Container + Container @@ -536,7 +554,7 @@ ALAS (Amazon) - Linux/FreeBSD + Linux/FreeBSD @@ -546,7 +564,7 @@ ALAS (Amazon) - Folder 6 + Folder 6 @@ -562,7 +580,7 @@ ALAS (Amazon) - Server + Server @@ -579,7 +597,7 @@ ALAS (Amazon) - Server + Server @@ -601,7 +619,7 @@ ALAS (Amazon) - results dir + results dir @@ -611,7 +629,7 @@ ALAS (Amazon) - Folder 7 + Folder 7 @@ -627,7 +645,7 @@ ALAS (Amazon) - JSON + JSON @@ -644,7 +662,7 @@ ALAS (Amazon) - JSON + JSON @@ -661,7 +679,7 @@ ALAS (Amazon) - JSON + JSON @@ -680,7 +698,7 @@ ALAS (Amazon) - + @@ -691,7 +709,7 @@ ALAS (Amazon) - Azure + Azure BLOB @@ -709,7 +727,7 @@ BLOB - .xml + .xml @@ -725,7 +743,7 @@ BLOB - .txt + .txt @@ -741,7 +759,7 @@ BLOB - .json + .json @@ -757,7 +775,7 @@ BLOB - .gz + .gz @@ -773,7 +791,7 @@ BLOB - Fetch + Fetch Vulnerability data @@ -792,7 +810,7 @@ Vulnerability data - HTTP + HTTP @@ -810,7 +828,7 @@ Vulnerability data - HTTP + HTTP @@ -828,14 +846,6 @@ Vulnerability data - WebUI - - - - - - - @@ -846,7 +856,7 @@ Vulnerability data - SSH + SSH @@ -864,7 +874,7 @@ Vulnerability data - SSH + SSH @@ -882,7 +892,7 @@ Vulnerability data - docker exec + docker exec lxc exec @@ -921,7 +931,7 @@ lxc exec - Insert + Insert @@ -933,31 +943,13 @@ lxc exec - - - - - - - Notify - - - - - - - - - - - - Select + Select @@ -969,7 +961,7 @@ lxc exec - + @@ -979,7 +971,7 @@ lxc exec - + @@ -989,7 +981,7 @@ lxc exec - + @@ -999,7 +991,7 @@ lxc exec - + @@ -1009,7 +1001,7 @@ lxc exec - + @@ -1019,7 +1011,7 @@ lxc exec - + @@ -1029,13 +1021,13 @@ lxc exec - + - Put + Put @@ -1047,7 +1039,7 @@ lxc exec - + @@ -1057,7 +1049,7 @@ lxc exec - + @@ -1067,13 +1059,13 @@ lxc exec - + - View Results + View Results on Terminal diff --git a/img/vuls-architecture.png b/img/vuls-architecture.png index 07305b9c..6872f195 100644 Binary files a/img/vuls-architecture.png and b/img/vuls-architecture.png differ diff --git a/img/vuls-scan-flow-fast.graphml b/img/vuls-scan-flow-fast.graphml new file mode 100644 index 00000000..ded5e2f4 --- /dev/null +++ b/img/vuls-scan-flow-fast.graphml @@ -0,0 +1,415 @@ + + + + + + + + + + + + + + + + + + + + + + + Detect the OS + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Get installed packages +Debian/Ubuntu: dpkg-query +Amazon/RHEL/CentOS: rpm +FreeBSD: pkg + + + + + + + + + + + + + + + + Write results to JSON files + + + + + + + + + + + + + + + + Get CVE IDs by using package manager +Amazon: yum plugin security +FreeBSD: pkg audit + + + + + + + + + + + + + + + + Report + + + + + + + + + + + + + + + + + + + Vulnerability Database + + + + + + + + + + Folder 1 + + + + + + + + + + + + + + + + CVE DB (NVD / JVN) + + + + + + + + + + + + + + + + OVAL DB + + + + + + + + + + + + + + + + + + Check upgradable packages +Debian/Ubuntu: apt-get upgrade --dry-run + + + + + + + + + + + + + + + + foreach +upgradable packages + + + + + + + + + + + + + + + + Parse changelog and get CVE IDs +Debian/Ubuntu: aptitude changelog + + + + + + + + + + + + + + + + end loop + + + + + + + + + + + + + + + + + + + + + + + + + + + + Amazon +FreeBSD + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CentOS +RHEL +Ubuntu +Debian +Oracle Linux + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Raspbian + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/vuls-scan-flow-fast.png b/img/vuls-scan-flow-fast.png new file mode 100644 index 00000000..24710d69 Binary files /dev/null and b/img/vuls-scan-flow-fast.png differ diff --git a/img/vuls-scan-flow.graphml b/img/vuls-scan-flow.graphml index 1409090b..39a67280 100644 --- a/img/vuls-scan-flow.graphml +++ b/img/vuls-scan-flow.graphml @@ -1,6 +1,6 @@ - + @@ -20,7 +20,7 @@ - Detect the OS + Detect the OS @@ -36,7 +36,7 @@ - + @@ -53,7 +53,7 @@ - Get installed packages + Get installed packages Debian/Ubuntu: dpkg-query Amazon/RHEL/CentOS: rpm FreeBSD: pkg @@ -72,7 +72,7 @@ FreeBSD: pkg - Check upgradable packages + Check upgradable packages Debian/Ubuntu: apt-get upgrade --dry-run @@ -89,7 +89,7 @@ Debian/Ubuntu: apt-get upgrade --dry-run - foreach + foreach upgradable packages @@ -106,7 +106,7 @@ upgradable packages - Parse changelog and get CVE IDs + Parse changelog and get CVE IDs Debian/Ubuntu: aptitude changelog @@ -123,7 +123,7 @@ Debian/Ubuntu: aptitude changelog - end loop + end loop @@ -139,7 +139,7 @@ Debian/Ubuntu: aptitude changelog - Select the CVE detail information + Write results to JSON files @@ -155,7 +155,7 @@ Debian/Ubuntu: aptitude changelog - Get CVE IDs by using package manager + Get CVE IDs by using package manager Amazon/RHEL: yum plugin security FreeBSD: pkg audit @@ -168,29 +168,12 @@ FreeBSD: pkg audit - - - - - - CVE DB (NVD / JVN) - - - - - - - - - - - Write results to JSON files -Reporting + Report @@ -200,14 +183,14 @@ Reporting - + - + - Get all changelogs of updatable packages at once -CentOS: yum update --changelog + Get all changelogs of updatable packages at once +yum changelog @@ -217,13 +200,13 @@ CentOS: yum update --changelog - + - + - Parse changelogs and get CVE IDs + Parse changelogs and get CVE IDs @@ -233,6 +216,87 @@ CentOS: yum update --changelog + + + + + + + Get all changelogs of updatable packages at once +Amazon / RHEL: yum changelog + + + + + + + + + + + + + + + + + + + + Vulnerability Database + + + + + + + + + + Folder 1 + + + + + + + + + + + + + + + + CVE DB (NVD / JVN) + + + + + + + + + + + + + + + + OVAL DB + + + + + + + + + + + @@ -251,8 +315,9 @@ CentOS: yum update --changelog - Debian -Ubuntu + Debian +Ubuntu +Raspbian @@ -314,7 +379,7 @@ Ubuntu - Amazon + Amazon RHEL FreeBSD @@ -328,17 +393,7 @@ FreeBSD - - - - - - - - - - - + @@ -348,7 +403,7 @@ FreeBSD - + @@ -358,29 +413,17 @@ FreeBSD - - - - - - - - - - - - - + - CentOS + CentOS - + @@ -388,7 +431,7 @@ FreeBSD - + @@ -398,11 +441,12 @@ FreeBSD - + + - - + + @@ -410,6 +454,39 @@ FreeBSD + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/vuls-scan-flow.png b/img/vuls-scan-flow.png index f4959b35..bd0d94c7 100644 Binary files a/img/vuls-scan-flow.png and b/img/vuls-scan-flow.png differ diff --git a/main.go b/main.go index 4e7ff891..1cb67cc9 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ import ( ) // Version of Vuls -var version = "0.3.0" +var version = "0.4.0" // Revision of Git var revision string diff --git a/models/cvecontents.go b/models/cvecontents.go new file mode 100644 index 00000000..170b3553 --- /dev/null +++ b/models/cvecontents.go @@ -0,0 +1,272 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "strings" + "time" +) + +// CveContents has CveContent +type CveContents map[CveContentType]CveContent + +// NewCveContents create CveContents +func NewCveContents(conts ...CveContent) CveContents { + m := CveContents{} + for _, cont := range conts { + m[cont.Type] = cont + } + return m +} + +// CveContentStr has CveContentType and Value +type CveContentStr struct { + Type CveContentType + Value string +} + +// Except returns CveContents except given keys for enumeration +func (v CveContents) Except(exceptCtypes ...CveContentType) (values CveContents) { + values = CveContents{} + for ctype, content := range v { + found := false + for _, exceptCtype := range exceptCtypes { + if ctype == exceptCtype { + found = true + break + } + } + if !found { + values[ctype] = content + } + } + return +} + +// SourceLinks returns link of source +func (v CveContents) SourceLinks(lang, myFamily, cveID string) (values []CveContentStr) { + if lang == "ja" { + if cont, found := v[JVN]; found && 0 < len(cont.SourceLink) { + values = append(values, CveContentStr{JVN, cont.SourceLink}) + } + } + + order := CveContentTypes{NVD, NewCveContentType(myFamily)} + for _, ctype := range order { + if cont, found := v[ctype]; found { + values = append(values, CveContentStr{ctype, cont.SourceLink}) + } + } + + if len(values) == 0 { + return []CveContentStr{{ + Type: NVD, + Value: "https://nvd.nist.gov/vuln/detail/" + cveID, + }} + } + return values +} + +/* +// Severities returns Severities +func (v CveContents) Severities(myFamily string) (values []CveContentStr) { + order := CveContentTypes{NVD, NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order)...)...) + + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.Severity) { + values = append(values, CveContentStr{ + Type: ctype, + Value: cont.Severity, + }) + } + } + return +} +*/ + +// CveContentCpes has CveContentType and Value +type CveContentCpes struct { + Type CveContentType + Value []Cpe +} + +// Cpes returns affected CPEs of this Vulnerability +func (v CveContents) Cpes(myFamily string) (values []CveContentCpes) { + order := CveContentTypes{NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order)...)...) + + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.Cpes) { + values = append(values, CveContentCpes{ + Type: ctype, + Value: cont.Cpes, + }) + } + } + return +} + +// CveContentRefs has CveContentType and Cpes +type CveContentRefs struct { + Type CveContentType + Value []Reference +} + +// References returns References +func (v CveContents) References(myFamily string) (values []CveContentRefs) { + order := CveContentTypes{NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order)...)...) + + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.References) { + values = append(values, CveContentRefs{ + Type: ctype, + Value: cont.References, + }) + } + } + return +} + +// CweIDs returns related CweIDs of the vulnerability +func (v CveContents) CweIDs(myFamily string) (values []CveContentStr) { + order := CveContentTypes{NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order)...)...) + + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.CweID) { + // RedHat's OVAL sometimes contains multiple CWE-IDs separated by spaces + for _, cweID := range strings.Fields(cont.CweID) { + values = append(values, CveContentStr{ + Type: ctype, + Value: cweID, + }) + } + } + } + return +} + +// CveContent has abstraction of various vulnerability information +type CveContent struct { + Type CveContentType + CveID string + Title string + Summary string + Severity string + Cvss2Score float64 + Cvss2Vector string + Cvss3Score float64 + Cvss3Vector string + SourceLink string + Cpes []Cpe + References References + CweID string + Published time.Time + LastModified time.Time +} + +// Empty checks the content is empty +func (c CveContent) Empty() bool { + return c.Summary == "" +} + +// CveContentType is a source of CVE information +type CveContentType string + +// NewCveContentType create CveContentType +func NewCveContentType(name string) CveContentType { + switch name { + case "nvd": + return NVD + case "jvn": + return JVN + case "redhat", "centos": + return RedHat + case "oracle": + return Oracle + case "ubuntu": + return Ubuntu + case "debian": + return Debian + default: + return Unknown + } +} + +const ( + // NVD is NVD + NVD CveContentType = "nvd" + + // JVN is JVN + JVN CveContentType = "jvn" + + // RedHat is RedHat + RedHat CveContentType = "redhat" + + // Debian is Debian + Debian CveContentType = "debian" + + // Ubuntu is Ubuntu + Ubuntu CveContentType = "ubuntu" + + // Oracle is Oracle Linux + Oracle CveContentType = "oracle" + + // Unknown is Unknown + Unknown CveContentType = "unknown" +) + +// CveContentTypes has slide of CveContentType +type CveContentTypes []CveContentType + +// AllCveContetTypes has all of CveContentTypes +var AllCveContetTypes = CveContentTypes{NVD, JVN, RedHat, Debian, Ubuntu} + +// Except returns CveContentTypes except for given args +func (c CveContentTypes) Except(excepts ...CveContentType) (excepted CveContentTypes) { + for _, ctype := range c { + found := false + for _, except := range excepts { + if ctype == except { + found = true + break + } + } + if !found { + excepted = append(excepted, ctype) + } + } + return +} + +// Cpe is Common Platform Enumeration +type Cpe struct { + CpeName string +} + +// References is a slice of Reference +type References []Reference + +// Reference has a related link of the CVE +type Reference struct { + Source string + Link string + RefID string +} diff --git a/models/cvecontents_test.go b/models/cvecontents_test.go new file mode 100644 index 00000000..4c3cadb2 --- /dev/null +++ b/models/cvecontents_test.go @@ -0,0 +1,206 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models + +import ( + "reflect" + "testing" +) + +func TestExcept(t *testing.T) { + var tests = []struct { + in CveContents + out CveContents + }{{ + in: CveContents{ + RedHat: {Type: RedHat}, + Ubuntu: {Type: Ubuntu}, + Debian: {Type: Debian}, + }, + out: CveContents{ + RedHat: {Type: RedHat}, + }, + }, + } + for _, tt := range tests { + actual := tt.in.Except(Ubuntu, Debian) + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestSourceLinks(t *testing.T) { + type in struct { + lang string + cveID string + cont CveContents + } + var tests = []struct { + in in + out []CveContentStr + }{ + // lang: ja + { + in: in{ + lang: "ja", + cveID: "CVE-2017-6074", + cont: CveContents{ + JVN: { + Type: JVN, + SourceLink: "https://jvn.jp/vu/JVNVU93610402/", + }, + RedHat: { + Type: RedHat, + SourceLink: "https://access.redhat.com/security/cve/CVE-2017-6074", + }, + NVD: { + Type: NVD, + SourceLink: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074", + }, + }, + }, + out: []CveContentStr{ + { + Type: JVN, + Value: "https://jvn.jp/vu/JVNVU93610402/", + }, + { + Type: NVD, + Value: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074", + }, + { + Type: RedHat, + Value: "https://access.redhat.com/security/cve/CVE-2017-6074", + }, + }, + }, + // lang: en + { + in: in{ + lang: "en", + cveID: "CVE-2017-6074", + cont: CveContents{ + JVN: { + Type: JVN, + SourceLink: "https://jvn.jp/vu/JVNVU93610402/", + }, + RedHat: { + Type: RedHat, + SourceLink: "https://access.redhat.com/security/cve/CVE-2017-6074", + }, + NVD: { + Type: NVD, + SourceLink: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074", + }, + }, + }, + out: []CveContentStr{ + { + Type: NVD, + Value: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074", + }, + { + Type: RedHat, + Value: "https://access.redhat.com/security/cve/CVE-2017-6074", + }, + }, + }, + // lang: empty + { + in: in{ + lang: "en", + cveID: "CVE-2017-6074", + cont: CveContents{}, + }, + out: []CveContentStr{ + { + Type: NVD, + Value: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074", + }, + }, + }, + } + for _, tt := range tests { + actual := tt.in.cont.SourceLinks(tt.in.lang, "redhat", tt.in.cveID) + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestVendorLink(t *testing.T) { + type in struct { + family string + vinfo VulnInfo + } + var tests = []struct { + in in + out map[string]string + }{ + { + in: in{ + family: "redhat", + vinfo: VulnInfo{ + CveID: "CVE-2017-6074", + CveContents: CveContents{ + JVN: { + Type: JVN, + SourceLink: "https://jvn.jp/vu/JVNVU93610402/", + }, + RedHat: { + Type: RedHat, + SourceLink: "https://access.redhat.com/security/cve/CVE-2017-6074", + }, + NVD: { + Type: NVD, + SourceLink: "https://nvd.nist.gov/vuln/detail/CVE-2017-6074", + }, + }, + }, + }, + out: map[string]string{ + "RHEL-CVE": "https://access.redhat.com/security/cve/CVE-2017-6074", + }, + }, + { + in: in{ + family: "ubuntu", + vinfo: VulnInfo{ + CveID: "CVE-2017-6074", + CveContents: CveContents{ + RedHat: { + Type: Ubuntu, + SourceLink: "https://access.redhat.com/security/cve/CVE-2017-6074", + }, + }, + }, + }, + out: map[string]string{ + "Ubuntu-CVE": "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2017-6074", + }, + }, + } + for _, tt := range tests { + actual := tt.in.vinfo.VendorLinks(tt.in.family) + for k := range tt.out { + if tt.out[k] != actual[k] { + t.Errorf("\nexpected: %s\n actual: %s\n", tt.out[k], actual[k]) + } + } + } +} diff --git a/models/models.go b/models/models.go index ddc94a3d..96850f24 100644 --- a/models/models.go +++ b/models/models.go @@ -17,535 +17,5 @@ along with this program. If not, see . package models -import ( - "fmt" - "sort" - "time" - - "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/cveapi" - cve "github.com/kotakanbe/go-cve-dictionary/models" -) - -// ScanHistory is the history of Scanning. -type ScanHistory struct { - ScanResults ScanResults -} - -// ScanResults is slice of ScanResult. -type ScanResults []ScanResult - -// Len implement Sort Interface -func (s ScanResults) Len() int { - return len(s) -} - -// Swap implement Sort Interface -func (s ScanResults) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -// Less implement Sort Interface -func (s ScanResults) Less(i, j int) bool { - if s[i].ServerName == s[j].ServerName { - return s[i].Container.ContainerID < s[i].Container.ContainerID - } - return s[i].ServerName < s[j].ServerName -} - -// ScanResult has the result of scanned CVE information. -type ScanResult struct { - ScannedAt time.Time - - Lang string - ServerName string // TOML Section key - Family string - Release string - Container Container - Platform Platform - - // Scanned Vulns via SSH + CPE Vulns - ScannedCves []VulnInfo - - KnownCves []CveInfo - UnknownCves []CveInfo - IgnoredCves []CveInfo - - Packages PackageInfoList - - Errors []string - Optional [][]interface{} -} - -// FillCveDetail fetches CVE detailed information from -// CVE Database, and then set to fields. -func (r ScanResult) FillCveDetail() (*ScanResult, error) { - set := map[string]VulnInfo{} - var cveIDs []string - for _, v := range r.ScannedCves { - set[v.CveID] = v - cveIDs = append(cveIDs, v.CveID) - } - - ds, err := cveapi.CveClient.FetchCveDetails(cveIDs) - if err != nil { - return nil, err - } - - known, unknown, ignored := CveInfos{}, CveInfos{}, CveInfos{} - for _, d := range ds { - cinfo := CveInfo{ - CveDetail: d, - VulnInfo: set[d.CveID], - } - cinfo.NilSliceToEmpty() - - // ignored - found := false - for _, icve := range config.Conf.Servers[r.ServerName].IgnoreCves { - if icve == d.CveID { - ignored = append(ignored, cinfo) - found = true - break - } - } - if found { - continue - } - - // unknown - if d.CvssScore(config.Conf.Lang) <= 0 { - unknown = append(unknown, cinfo) - continue - } - - // known - known = append(known, cinfo) - } - sort.Sort(known) - sort.Sort(unknown) - sort.Sort(ignored) - r.KnownCves = known - r.UnknownCves = unknown - r.IgnoredCves = ignored - return &r, nil -} - -// FilterByCvssOver is filter function. -func (r ScanResult) FilterByCvssOver() ScanResult { - cveInfos := []CveInfo{} - for _, cveInfo := range r.KnownCves { - if config.Conf.CvssScoreOver < cveInfo.CveDetail.CvssScore(config.Conf.Lang) { - cveInfos = append(cveInfos, cveInfo) - } - } - r.KnownCves = cveInfos - return r -} - -// ReportFileName returns the filename on localhost without extention -func (r ScanResult) ReportFileName() (name string) { - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s", r.ServerName) - } - return fmt.Sprintf("%s@%s", r.Container.Name, r.ServerName) -} - -// ReportKeyName returns the name of key on S3, Azure-Blob without extention -func (r ScanResult) ReportKeyName() (name string) { - timestr := r.ScannedAt.Format(time.RFC3339) - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s/%s", timestr, r.ServerName) - } - return fmt.Sprintf("%s/%s@%s", timestr, r.Container.Name, r.ServerName) -} - -// ServerInfo returns server name one line -func (r ScanResult) ServerInfo() string { - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s (%s%s)", - r.ServerName, r.Family, r.Release) - } - return fmt.Sprintf( - "%s / %s (%s%s) on %s", - r.Container.Name, - r.Container.ContainerID, - r.Family, - r.Release, - r.ServerName, - ) -} - -// ServerInfoTui returns server infromation for TUI sidebar -func (r ScanResult) ServerInfoTui() string { - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s (%s%s)", - r.ServerName, r.Family, r.Release) - } - return fmt.Sprintf( - "|-- %s (%s%s)", - r.Container.Name, - r.Family, - r.Release, - // r.Container.ContainerID, - ) -} - -// FormatServerName returns server and container name -func (r ScanResult) FormatServerName() string { - if len(r.Container.ContainerID) == 0 { - return r.ServerName - } - return fmt.Sprintf("%s@%s", - r.Container.Name, r.ServerName) -} - -// CveSummary summarize the number of CVEs group by CVSSv2 Severity -func (r ScanResult) CveSummary() string { - var high, medium, low, unknown int - cves := append(r.KnownCves, r.UnknownCves...) - for _, cveInfo := range cves { - score := cveInfo.CveDetail.CvssScore(config.Conf.Lang) - switch { - case 7.0 <= score: - high++ - case 4.0 <= score: - medium++ - case 0 < score: - low++ - default: - unknown++ - } - } - - if config.Conf.IgnoreUnscoredCves { - return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d)", - high+medium+low, high, medium, low) - } - return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d ?:%d)", - high+medium+low+unknown, high, medium, low, unknown) -} - -// AllCves returns Known and Unknown CVEs -func (r ScanResult) AllCves() []CveInfo { - return append(r.KnownCves, r.UnknownCves...) -} - -// NWLink has network link information. -type NWLink struct { - IPAddress string - Netmask string - DevName string - LinkState string -} - -// Confidence is a ranking how confident the CVE-ID was deteted correctly -// Score: 0 - 100 -type Confidence struct { - Score int - DetectionMethod string -} - -func (c Confidence) String() string { - return fmt.Sprintf("%d / %s", c.Score, c.DetectionMethod) -} - -const ( - // CpeNameMatchStr is a String representation of CpeNameMatch - CpeNameMatchStr = "CpeNameMatch" - - // YumUpdateSecurityMatchStr is a String representation of YumUpdateSecurityMatch - YumUpdateSecurityMatchStr = "YumUpdateSecurityMatch" - - // PkgAuditMatchStr is a String representation of PkgAuditMatch - PkgAuditMatchStr = "PkgAuditMatch" - - // ChangelogExactMatchStr is a String representation of ChangelogExactMatch - ChangelogExactMatchStr = "ChangelogExactMatch" - - // ChangelogLenientMatchStr is a String representation of ChangelogLenientMatch - ChangelogLenientMatchStr = "ChangelogLenientMatch" - - // FailedToGetChangelog is a String representation of FailedToGetChangelog - FailedToGetChangelog = "FailedToGetChangelog" - - // FailedToFindVersionInChangelog is a String representation of FailedToFindVersionInChangelog - FailedToFindVersionInChangelog = "FailedToFindVersionInChangelog" -) - -// CpeNameMatch is a ranking how confident the CVE-ID was deteted correctly -var CpeNameMatch = Confidence{100, CpeNameMatchStr} - -// YumUpdateSecurityMatch is a ranking how confident the CVE-ID was deteted correctly -var YumUpdateSecurityMatch = Confidence{100, YumUpdateSecurityMatchStr} - -// PkgAuditMatch is a ranking how confident the CVE-ID was deteted correctly -var PkgAuditMatch = Confidence{100, PkgAuditMatchStr} - -// ChangelogExactMatch is a ranking how confident the CVE-ID was deteted correctly -var ChangelogExactMatch = Confidence{95, ChangelogExactMatchStr} - -// ChangelogLenientMatch is a ranking how confident the CVE-ID was deteted correctly -var ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr} - -// VulnInfos is VulnInfo list, getter/setter, sortable methods. -type VulnInfos []VulnInfo - -// VulnInfo holds a vulnerability information and unsecure packages -type VulnInfo struct { - CveID string - Confidence Confidence - Packages PackageInfoList - DistroAdvisories []DistroAdvisory // for Aamazon, RHEL, FreeBSD - CpeNames []string -} - -// NilSliceToEmpty set nil slice fields to empty slice to avoid null in JSON -func (v *VulnInfo) NilSliceToEmpty() { - if v.CpeNames == nil { - v.CpeNames = []string{} - } - if v.DistroAdvisories == nil { - v.DistroAdvisories = []DistroAdvisory{} - } - if v.Packages == nil { - v.Packages = PackageInfoList{} - } -} - -// FindByCveID find by CVEID -func (s VulnInfos) FindByCveID(cveID string) (VulnInfo, bool) { - for _, p := range s { - if cveID == p.CveID { - return p, true - } - } - return VulnInfo{CveID: cveID}, false -} - -// immutable -func (s VulnInfos) set(cveID string, v VulnInfo) VulnInfos { - for i, p := range s { - if cveID == p.CveID { - s[i] = v - return s - } - } - return append(s, v) -} - -// Len implement Sort Interface -func (s VulnInfos) Len() int { - return len(s) -} - -// Swap implement Sort Interface -func (s VulnInfos) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -// Less implement Sort Interface -func (s VulnInfos) Less(i, j int) bool { - return s[i].CveID < s[j].CveID -} - -// CveInfos is for sorting -type CveInfos []CveInfo - -func (c CveInfos) Len() int { - return len(c) -} - -func (c CveInfos) Swap(i, j int) { - c[i], c[j] = c[j], c[i] -} - -func (c CveInfos) Less(i, j int) bool { - lang := config.Conf.Lang - if c[i].CveDetail.CvssScore(lang) == c[j].CveDetail.CvssScore(lang) { - return c[i].CveDetail.CveID < c[j].CveDetail.CveID - } - return c[j].CveDetail.CvssScore(lang) < c[i].CveDetail.CvssScore(lang) -} - -// CveInfo has Cve Information. -type CveInfo struct { - CveDetail cve.CveDetail - VulnInfo -} - -// NilSliceToEmpty set nil slice fields to empty slice to avoid null in JSON -func (c *CveInfo) NilSliceToEmpty() { - if c.CveDetail.Nvd.Cpes == nil { - c.CveDetail.Nvd.Cpes = []cve.Cpe{} - } - if c.CveDetail.Jvn.Cpes == nil { - c.CveDetail.Jvn.Cpes = []cve.Cpe{} - } - if c.CveDetail.Nvd.References == nil { - c.CveDetail.Nvd.References = []cve.Reference{} - } - if c.CveDetail.Jvn.References == nil { - c.CveDetail.Jvn.References = []cve.Reference{} - } -} - -// PackageInfoList is slice of PackageInfo -type PackageInfoList []PackageInfo - -// Exists returns true if exists the name -func (ps PackageInfoList) Exists(name string) bool { - for _, p := range ps { - if p.Name == name { - return true - } - } - return false -} - -// UniqByName be uniq by name. -func (ps PackageInfoList) UniqByName() (distincted PackageInfoList) { - set := make(map[string]PackageInfo) - for _, p := range ps { - set[p.Name] = p - } - //sort by key - keys := []string{} - for key := range set { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - distincted = append(distincted, set[key]) - } - return -} - -// FindByName search PackageInfo by name -func (ps PackageInfoList) FindByName(name string) (result PackageInfo, found bool) { - for _, p := range ps { - if p.Name == name { - return p, true - } - } - return PackageInfo{}, false -} - -// MergeNewVersion merges candidate version information to the receiver struct -func (ps PackageInfoList) MergeNewVersion(as PackageInfoList) { - for _, a := range as { - for i, p := range ps { - if p.Name == a.Name { - ps[i].NewVersion = a.NewVersion - ps[i].NewRelease = a.NewRelease - } - } - } -} - -func (ps PackageInfoList) countUpdatablePacks() int { - count := 0 - set := make(map[string]bool) - for _, p := range ps { - if len(p.NewVersion) != 0 && !set[p.Name] { - count++ - set[p.Name] = true - } - } - return count -} - -// FormatUpdatablePacksSummary returns a summary of updatable packages -func (ps PackageInfoList) FormatUpdatablePacksSummary() string { - return fmt.Sprintf("%d updatable packages", - ps.countUpdatablePacks()) -} - -// Find search PackageInfo by name-version-release -// func (ps PackageInfoList) find(nameVersionRelease string) (PackageInfo, bool) { -// for _, p := range ps { -// joined := p.Name -// if 0 < len(p.Version) { -// joined = fmt.Sprintf("%s-%s", joined, p.Version) -// } -// if 0 < len(p.Release) { -// joined = fmt.Sprintf("%s-%s", joined, p.Release) -// } -// if joined == nameVersionRelease { -// return p, true -// } -// } -// return PackageInfo{}, false -// } - -// PackageInfosByName implements sort.Interface for []PackageInfo based on -// the Name field. -type PackageInfosByName []PackageInfo - -func (a PackageInfosByName) Len() int { return len(a) } -func (a PackageInfosByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a PackageInfosByName) Less(i, j int) bool { return a[i].Name < a[j].Name } - -// PackageInfo has installed packages. -type PackageInfo struct { - Name string - Version string - Release string - NewVersion string - NewRelease string - Repository string - Changelog Changelog -} - -// Changelog has contents of changelog and how to get it. -// Method: modesl.detectionMethodStr -type Changelog struct { - Contents string - Method string -} - -// ToStringCurrentVersion returns package name-version-release -func (p PackageInfo) ToStringCurrentVersion() string { - str := p.Name - if 0 < len(p.Version) { - str = fmt.Sprintf("%s-%s", str, p.Version) - } - if 0 < len(p.Release) { - str = fmt.Sprintf("%s-%s", str, p.Release) - } - return str -} - -// ToStringNewVersion returns package name-version-release -func (p PackageInfo) ToStringNewVersion() string { - str := p.Name - if 0 < len(p.NewVersion) { - str = fmt.Sprintf("%s-%s", str, p.NewVersion) - } - if 0 < len(p.NewRelease) { - str = fmt.Sprintf("%s-%s", str, p.NewRelease) - } - return str -} - -// DistroAdvisory has Amazon Linux, RHEL, FreeBSD Security Advisory information. -type DistroAdvisory struct { - AdvisoryID string - Severity string - Issued time.Time - Updated time.Time -} - -// Container has Container information -type Container struct { - ContainerID string - Name string - Image string - Type string -} - -// Platform has platform information -type Platform struct { - Name string // aws or azure or gcp or other... - InstanceID string -} +// JSONVersion is JSON Version +const JSONVersion = 2 diff --git a/models/models_test.go b/models/models_test.go index 0ef1d40e..aee32778 100644 --- a/models/models_test.go +++ b/models/models_test.go @@ -16,122 +16,3 @@ along with this program. If not, see . */ package models - -import ( - "reflect" - "testing" - - "github.com/k0kubun/pp" -) - -func TestPackageInfoListUniqByName(t *testing.T) { - var test = struct { - in PackageInfoList - out PackageInfoList - }{ - PackageInfoList{ - { - Name: "hoge", - }, - { - Name: "fuga", - }, - { - Name: "hoge", - }, - }, - PackageInfoList{ - { - Name: "hoge", - }, - { - Name: "fuga", - }, - }, - } - - actual := test.in.UniqByName() - for i, ePack := range test.out { - if actual[i].Name == ePack.Name { - t.Errorf("expected %#v, actual %#v", ePack.Name, actual[i].Name) - } - } -} - -func TestMergeNewVersion(t *testing.T) { - var test = struct { - a PackageInfoList - b PackageInfoList - expected PackageInfoList - }{ - PackageInfoList{ - { - Name: "hoge", - }, - }, - PackageInfoList{ - { - Name: "hoge", - NewVersion: "1.0.0", - NewRelease: "release1", - }, - }, - PackageInfoList{ - { - Name: "hoge", - NewVersion: "1.0.0", - NewRelease: "release1", - }, - }, - } - - test.a.MergeNewVersion(test.b) - if !reflect.DeepEqual(test.a, test.expected) { - e := pp.Sprintf("%v", test.a) - a := pp.Sprintf("%v", test.expected) - t.Errorf("expected %s, actual %s", e, a) - } -} -func TestVulnInfosSetGet(t *testing.T) { - var test = struct { - in []string - out []string - }{ - []string{ - "CVE1", - "CVE2", - "CVE3", - "CVE1", - "CVE1", - "CVE2", - "CVE3", - }, - []string{ - "CVE1", - "CVE2", - "CVE3", - }, - } - - // var ps packageCveInfos - var ps VulnInfos - for _, cid := range test.in { - ps = ps.set(cid, VulnInfo{CveID: cid}) - } - - if len(test.out) != len(ps) { - t.Errorf("length: expected %d, actual %d", len(test.out), len(ps)) - } - - for i, expectedCid := range test.out { - if expectedCid != ps[i].CveID { - t.Errorf("expected %s, actual %s", expectedCid, ps[i].CveID) - } - } - for _, cid := range test.in { - p, _ := ps.FindByCveID(cid) - if p.CveID != cid { - t.Errorf("expected %s, actual %s", cid, p.CveID) - } - } -} diff --git a/models/packages.go b/models/packages.go new file mode 100644 index 00000000..473d52bd --- /dev/null +++ b/models/packages.go @@ -0,0 +1,153 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "bytes" + "fmt" + "strings" +) + +// Packages is Map of Package +// { "package-name": Package } +type Packages map[string]Package + +// NewPackages create Packages +func NewPackages(packs ...Package) Packages { + m := Packages{} + for _, pack := range packs { + m[pack.Name] = pack + } + return m +} + +// MergeNewVersion merges candidate version information to the receiver struct +func (ps Packages) MergeNewVersion(as Packages) { + for _, a := range as { + if pack, ok := ps[a.Name]; ok { + pack.NewVersion = a.NewVersion + pack.NewRelease = a.NewRelease + pack.Repository = a.Repository + ps[a.Name] = pack + } + } +} + +// Merge returns merged map (immutable) +func (ps Packages) Merge(other Packages) Packages { + merged := Packages{} + for k, v := range ps { + merged[k] = v + } + for k, v := range other { + merged[k] = v + } + return merged +} + +// FormatUpdatablePacksSummary returns a summary of updatable packages +func (ps Packages) FormatUpdatablePacksSummary() string { + nUpdatable := 0 + for _, p := range ps { + if p.NewVersion != "" { + nUpdatable++ + } + } + return fmt.Sprintf("%d updatable packages", nUpdatable) +} + +// FindOne search a element by name-newver-newrel-arch +func (ps Packages) FindOne(f func(Package) bool) (string, Package, bool) { + for key, p := range ps { + if f(p) { + return key, p, true + } + } + return "", Package{}, false +} + +// Package has installed packages. +type Package struct { + Name string + Version string + Release string + NewVersion string + NewRelease string + Arch string + Repository string + Changelog Changelog +} + +// FormatVer returns package version-release +func (p Package) FormatVer() string { + ver := p.Version + if 0 < len(p.Release) { + ver = fmt.Sprintf("%s-%s", ver, p.Release) + } + return ver +} + +// FormatNewVer returns package version-release +func (p Package) FormatNewVer() string { + ver := p.NewVersion + if 0 < len(p.NewRelease) { + ver = fmt.Sprintf("%s-%s", ver, p.NewRelease) + } + return ver +} + +// FormatVersionFromTo formats installed and new package version +func (p Package) FormatVersionFromTo(notFixedYet bool) string { + to := p.FormatNewVer() + if notFixedYet { + to = "Not Fixed Yet" + } + return fmt.Sprintf("%s-%s -> %s", p.Name, p.FormatVer(), to) +} + +// FormatChangelog formats the changelog +func (p Package) FormatChangelog() string { + buf := []string{} + packVer := fmt.Sprintf("%s-%s -> %s", + p.Name, p.FormatVer(), p.FormatNewVer()) + var delim bytes.Buffer + for i := 0; i < len(packVer); i++ { + delim.WriteString("-") + } + + clog := p.Changelog.Contents + if lines := strings.Split(clog, "\n"); len(lines) != 0 { + clog = strings.Join(lines[0:len(lines)-1], "\n") + } + + switch p.Changelog.Method { + case FailedToGetChangelog: + clog = "No changelogs" + case FailedToFindVersionInChangelog: + clog = "Failed to parse changelogs. For detials, check yourself" + } + buf = append(buf, packVer, delim.String(), clog) + return strings.Join(buf, "\n") +} + +// Changelog has contents of changelog and how to get it. +// Method: models.detectionMethodStr +type Changelog struct { + Contents string + Method DetectionMethod +} diff --git a/models/packages_test.go b/models/packages_test.go new file mode 100644 index 00000000..48eb99d8 --- /dev/null +++ b/models/packages_test.go @@ -0,0 +1,89 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models + +import ( + "reflect" + "testing" + + "github.com/k0kubun/pp" +) + +func TestMergeNewVersion(t *testing.T) { + var test = struct { + a Packages + b Packages + expected Packages + }{ + Packages{ + "hoge": { + Name: "hoge", + }, + }, + Packages{ + "hoge": { + Name: "hoge", + NewVersion: "1.0.0", + NewRelease: "release1", + }, + }, + Packages{ + "hoge": { + Name: "hoge", + NewVersion: "1.0.0", + NewRelease: "release1", + }, + }, + } + + test.a.MergeNewVersion(test.b) + if !reflect.DeepEqual(test.a, test.expected) { + e := pp.Sprintf("%v", test.a) + a := pp.Sprintf("%v", test.expected) + t.Errorf("expected %s, actual %s", e, a) + } +} + +func TestMerge(t *testing.T) { + var test = struct { + a Packages + b Packages + expected Packages + }{ + Packages{ + "hoge": {Name: "hoge"}, + "fuga": {Name: "fuga"}, + }, + Packages{ + "hega": {Name: "hega"}, + "hage": {Name: "hage"}, + }, + Packages{ + "hoge": {Name: "hoge"}, + "fuga": {Name: "fuga"}, + "hega": {Name: "hega"}, + "hage": {Name: "hage"}, + }, + } + + actual := test.a.Merge(test.b) + if !reflect.DeepEqual(actual, test.expected) { + e := pp.Sprintf("%v", test.expected) + a := pp.Sprintf("%v", actual) + t.Errorf("expected %s, actual %s", e, a) + } +} diff --git a/models/scanresults.go b/models/scanresults.go new file mode 100644 index 00000000..954ec8ed --- /dev/null +++ b/models/scanresults.go @@ -0,0 +1,191 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "bytes" + "fmt" + "time" + + "github.com/future-architect/vuls/config" +) + +// ScanResults is a slide of ScanResult +type ScanResults []ScanResult + +// ScanResult has the result of scanned CVE information. +type ScanResult struct { + ScannedAt time.Time + ReportedAt time.Time + JSONVersion int + Lang string + ServerUUID string + ServerName string // TOML Section key + Family string + Release string + Container Container + Platform Platform + + // Scanned Vulns by SSH scan + CPE + OVAL + ScannedCves VulnInfos + + RunningKernel Kernel + Packages Packages + Errors []string + Optional [][]interface{} + + Config struct { + Scan config.Config + Report config.Config + } +} + +// Kernel has the Release, version and whether need restart +type Kernel struct { + Release string + Version string + RebootRequired bool +} + +// FilterByCvssOver is filter function. +func (r ScanResult) FilterByCvssOver(over float64) ScanResult { + filtered := r.ScannedCves.Find(func(v VulnInfo) bool { + v2Max := v.MaxCvss2Score() + v3Max := v.MaxCvss3Score() + max := v2Max.Value.Score + if max < v3Max.Value.Score { + max = v3Max.Value.Score + } + if over <= max { + return true + } + return false + }) + + copiedScanResult := r + copiedScanResult.ScannedCves = filtered + return copiedScanResult +} + +// FilterIgnoreCves is filter function. +func (r ScanResult) FilterIgnoreCves(cveIDs []string) ScanResult { + filtered := r.ScannedCves.Find(func(v VulnInfo) bool { + for _, c := range cveIDs { + if v.CveID == c { + return false + } + } + return true + }) + copiedScanResult := r + copiedScanResult.ScannedCves = filtered + return copiedScanResult +} + +// ReportFileName returns the filename on localhost without extention +func (r ScanResult) ReportFileName() (name string) { + if len(r.Container.ContainerID) == 0 { + return fmt.Sprintf("%s", r.ServerName) + } + return fmt.Sprintf("%s@%s", r.Container.Name, r.ServerName) +} + +// ReportKeyName returns the name of key on S3, Azure-Blob without extention +func (r ScanResult) ReportKeyName() (name string) { + timestr := r.ScannedAt.Format(time.RFC3339) + if len(r.Container.ContainerID) == 0 { + return fmt.Sprintf("%s/%s", timestr, r.ServerName) + } + return fmt.Sprintf("%s/%s@%s", timestr, r.Container.Name, r.ServerName) +} + +// ServerInfo returns server name one line +func (r ScanResult) ServerInfo() string { + if len(r.Container.ContainerID) == 0 { + return fmt.Sprintf("%s (%s%s)", + r.FormatServerName(), r.Family, r.Release) + } + return fmt.Sprintf( + "%s (%s%s) on %s", + r.FormatServerName(), + r.Family, + r.Release, + r.ServerName, + ) +} + +// ServerInfoTui returns server infromation for TUI sidebar +func (r ScanResult) ServerInfoTui() string { + if len(r.Container.ContainerID) == 0 { + line := fmt.Sprintf("%s (%s%s)", + r.ServerName, r.Family, r.Release) + if r.RunningKernel.RebootRequired { + return "[Reboot] " + line + } + return line + } + + fmtstr := "|-- %s (%s%s)" + if r.RunningKernel.RebootRequired { + fmtstr = "|-- [Reboot] %s (%s%s)" + } + return fmt.Sprintf(fmtstr, r.Container.Name, r.Family, r.Release) +} + +// FormatServerName returns server and container name +func (r ScanResult) FormatServerName() (name string) { + if len(r.Container.ContainerID) == 0 { + name = r.ServerName + } else { + name = fmt.Sprintf("%s@%s", + r.Container.Name, r.ServerName) + } + if r.RunningKernel.RebootRequired { + name = "[Reboot Required] " + name + } + return +} + +// FormatTextReportHeadedr returns header of text report +func (r ScanResult) FormatTextReportHeadedr() string { + serverInfo := r.ServerInfo() + var buf bytes.Buffer + for i := 0; i < len(serverInfo); i++ { + buf.WriteString("=") + } + return fmt.Sprintf("%s\n%s\n%s\t%s\n", + r.ServerInfo(), + buf.String(), + r.ScannedCves.FormatCveSummary(), + r.Packages.FormatUpdatablePacksSummary(), + ) +} + +// Container has Container information +type Container struct { + ContainerID string + Name string + Image string + Type string +} + +// Platform has platform information +type Platform struct { + Name string // aws or azure or gcp or other... + InstanceID string +} diff --git a/models/scanresults_test.go b/models/scanresults_test.go new file mode 100644 index 00000000..9df1a52b --- /dev/null +++ b/models/scanresults_test.go @@ -0,0 +1,257 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models + +import ( + "reflect" + "testing" + "time" + + "github.com/k0kubun/pp" +) + +func TestFilterByCvssOver(t *testing.T) { + type in struct { + over float64 + rs ScanResult + } + var tests = []struct { + in in + out ScanResult + }{ + { + in: in{ + over: 7.0, + rs: ScanResult{ + ScannedCves: VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: NewCveContents( + CveContent{ + Type: NVD, + CveID: "CVE-2017-0001", + Cvss2Score: 7.1, + LastModified: time.Time{}, + }, + ), + }, + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: NewCveContents( + CveContent{ + Type: NVD, + CveID: "CVE-2017-0002", + Cvss2Score: 6.9, + LastModified: time.Time{}, + }, + ), + }, + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: NewCveContents( + CveContent{ + Type: NVD, + CveID: "CVE-2017-0003", + Cvss2Score: 6.9, + LastModified: time.Time{}, + }, + CveContent{ + Type: JVN, + CveID: "CVE-2017-0003", + Cvss2Score: 7.2, + LastModified: time.Time{}, + }, + ), + }, + }, + }, + }, + out: ScanResult{ + ScannedCves: VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: NewCveContents( + CveContent{ + Type: NVD, + CveID: "CVE-2017-0001", + Cvss2Score: 7.1, + LastModified: time.Time{}, + }, + ), + }, + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: NewCveContents( + CveContent{ + Type: NVD, + CveID: "CVE-2017-0003", + Cvss2Score: 6.9, + LastModified: time.Time{}, + }, + CveContent{ + Type: JVN, + CveID: "CVE-2017-0003", + Cvss2Score: 7.2, + LastModified: time.Time{}, + }, + ), + }, + }, + }, + }, + // OVAL Severity + { + in: in{ + over: 7.0, + rs: ScanResult{ + ScannedCves: VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: NewCveContents( + CveContent{ + Type: Ubuntu, + CveID: "CVE-2017-0001", + Severity: "HIGH", + LastModified: time.Time{}, + }, + ), + }, + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: NewCveContents( + CveContent{ + Type: RedHat, + CveID: "CVE-2017-0002", + Severity: "CRITICAL", + LastModified: time.Time{}, + }, + ), + }, + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: NewCveContents( + CveContent{ + Type: Oracle, + CveID: "CVE-2017-0003", + Severity: "IMPORTANT", + LastModified: time.Time{}, + }, + ), + }, + }, + }, + }, + out: ScanResult{ + ScannedCves: VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: NewCveContents( + CveContent{ + Type: Ubuntu, + CveID: "CVE-2017-0001", + Severity: "HIGH", + LastModified: time.Time{}, + }, + ), + }, + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: NewCveContents( + CveContent{ + Type: RedHat, + CveID: "CVE-2017-0002", + Severity: "CRITICAL", + LastModified: time.Time{}, + }, + ), + }, + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: NewCveContents( + CveContent{ + Type: Oracle, + CveID: "CVE-2017-0003", + Severity: "IMPORTANT", + LastModified: time.Time{}, + }, + ), + }, + }, + }, + }, + } + for _, tt := range tests { + actual := tt.in.rs.FilterByCvssOver(tt.in.over) + for k := range tt.out.ScannedCves { + if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { + o := pp.Sprintf("%v", tt.out.ScannedCves[k]) + a := pp.Sprintf("%v", actual.ScannedCves[k]) + t.Errorf("[%s] expected: %v\n actual: %v\n", k, o, a) + } + } + } +} + +func TestFilterIgnoreCveIDs(t *testing.T) { + type in struct { + cves []string + rs ScanResult + } + var tests = []struct { + in in + out ScanResult + }{ + { + in: in{ + cves: []string{"CVE-2017-0002"}, + rs: ScanResult{ + ScannedCves: VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + }, + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + }, + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + }, + }, + }, + }, + out: ScanResult{ + ScannedCves: VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + }, + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + }, + }, + }, + }, + } + for _, tt := range tests { + actual := tt.in.rs.FilterIgnoreCves(tt.in.cves) + for k := range tt.out.ScannedCves { + if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { + o := pp.Sprintf("%v", tt.out.ScannedCves[k]) + a := pp.Sprintf("%v", actual.ScannedCves[k]) + t.Errorf("[%s] expected: %v\n actual: %v\n", k, o, a) + } + } + } +} diff --git a/models/utils.go b/models/utils.go new file mode 100644 index 00000000..d1b97308 --- /dev/null +++ b/models/utils.go @@ -0,0 +1,114 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "fmt" + "strings" + + cvedict "github.com/kotakanbe/go-cve-dictionary/models" +) + +// ConvertNvdToModel convert NVD to CveContent +func ConvertNvdToModel(cveID string, nvd cvedict.Nvd) *CveContent { + var cpes []Cpe + for _, c := range nvd.Cpes { + cpes = append(cpes, Cpe{CpeName: c.CpeName}) + } + + var refs []Reference + for _, r := range nvd.References { + refs = append(refs, Reference{ + Link: r.Link, + Source: r.Source, + }) + } + + validVec := true + for _, v := range []string{ + nvd.AccessVector, + nvd.AccessComplexity, + nvd.Authentication, + nvd.ConfidentialityImpact, + nvd.IntegrityImpact, + nvd.AvailabilityImpact, + } { + if len(v) == 0 { + validVec = false + } + } + + vector := "" + if validVec { + vector = fmt.Sprintf("AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s", + string(nvd.AccessVector[0]), + string(nvd.AccessComplexity[0]), + string(nvd.Authentication[0]), + string(nvd.ConfidentialityImpact[0]), + string(nvd.IntegrityImpact[0]), + string(nvd.AvailabilityImpact[0])) + } + + //TODO CVSSv3 + return &CveContent{ + Type: NVD, + CveID: cveID, + Summary: nvd.Summary, + Cvss2Score: nvd.Score, + Cvss2Vector: vector, + Severity: "", // severity is not contained in NVD + SourceLink: "https://nvd.nist.gov/vuln/detail/" + cveID, + Cpes: cpes, + CweID: nvd.CweID, + References: refs, + Published: nvd.PublishedDate, + LastModified: nvd.LastModifiedDate, + } +} + +// ConvertJvnToModel convert JVN to CveContent +func ConvertJvnToModel(cveID string, jvn cvedict.Jvn) *CveContent { + var cpes []Cpe + for _, c := range jvn.Cpes { + cpes = append(cpes, Cpe{CpeName: c.CpeName}) + } + + refs := []Reference{} + for _, r := range jvn.References { + refs = append(refs, Reference{ + Link: r.Link, + Source: r.Source, + }) + } + + vector := strings.TrimSuffix(strings.TrimPrefix(jvn.Vector, "("), ")") + return &CveContent{ + Type: JVN, + CveID: cveID, + Title: jvn.Title, + Summary: jvn.Summary, + Severity: jvn.Severity, + Cvss2Score: jvn.Score, + Cvss2Vector: vector, + SourceLink: jvn.JvnLink, + Cpes: cpes, + References: refs, + Published: jvn.PublishedDate, + LastModified: jvn.LastModifiedDate, + } +} diff --git a/models/vulninfos.go b/models/vulninfos.go new file mode 100644 index 00000000..b7451231 --- /dev/null +++ b/models/vulninfos.go @@ -0,0 +1,666 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "bytes" + "fmt" + "sort" + "strings" + "time" + + "github.com/future-architect/vuls/config" +) + +// VulnInfos has a map of VulnInfo +// Key: CveID +type VulnInfos map[string]VulnInfo + +// Find elements that matches the function passed in argument +func (v VulnInfos) Find(f func(VulnInfo) bool) VulnInfos { + filtered := VulnInfos{} + for _, vv := range v { + if f(vv) { + filtered[vv.CveID] = vv + } + } + return filtered +} + +// FindScoredVulns return scored vulnerabilities +func (v VulnInfos) FindScoredVulns() VulnInfos { + return v.Find(func(vv VulnInfo) bool { + if 0 < vv.MaxCvss2Score().Value.Score || + 0 < vv.MaxCvss3Score().Value.Score { + return true + } + return false + }) +} + +// ToSortedSlice returns slice of VulnInfos that is sorted by Score, CVE-ID +func (v VulnInfos) ToSortedSlice() (sorted []VulnInfo) { + for k := range v { + sorted = append(sorted, v[k]) + } + sort.Slice(sorted, func(i, j int) bool { + maxI := sorted[i].MaxCvssScore() + maxJ := sorted[j].MaxCvssScore() + if maxI.Value.Score != maxJ.Value.Score { + return maxJ.Value.Score < maxI.Value.Score + } + return sorted[i].CveID < sorted[j].CveID + }) + return +} + +// CountGroupBySeverity summarize the number of CVEs group by CVSSv2 Severity +func (v VulnInfos) CountGroupBySeverity() map[string]int { + m := map[string]int{} + for _, vInfo := range v { + score := vInfo.MaxCvss2Score().Value.Score + if score < 0.1 { + score = vInfo.MaxCvss3Score().Value.Score + } + switch { + case 7.0 <= score: + m["High"]++ + case 4.0 <= score: + m["Medium"]++ + case 0 < score: + m["Low"]++ + default: + m["Unknown"]++ + } + } + return m +} + +// FormatCveSummary summarize the number of CVEs group by CVSSv2 Severity +func (v VulnInfos) FormatCveSummary() string { + m := v.CountGroupBySeverity() + + if config.Conf.IgnoreUnscoredCves { + return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d)", + m["High"]+m["Medium"]+m["Low"], m["High"], m["Medium"], m["Low"]) + } + return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d ?:%d)", + m["High"]+m["Medium"]+m["Low"]+m["Unknown"], + m["High"], m["Medium"], m["Low"], m["Unknown"]) +} + +// PackageStatuses is a list of PackageStatus +type PackageStatuses []PackageStatus + +// Sort by Name +func (p PackageStatuses) Sort() { + sort.Slice(p, func(i, j int) bool { + return p[i].Name < p[j].Name + }) + return +} + +// PackageStatus has name and other status abount the package +type PackageStatus struct { + Name string + NotFixedYet bool +} + +// VulnInfo has a vulnerability information and unsecure packages +type VulnInfo struct { + CveID string + Confidence Confidence + AffectedPackages PackageStatuses + DistroAdvisories []DistroAdvisory // for Aamazon, RHEL, FreeBSD + CpeNames []string + CveContents CveContents +} + +// Titles returns tilte (TUI) +func (v VulnInfo) Titles(lang, myFamily string) (values []CveContentStr) { + if lang == "ja" { + if cont, found := v.CveContents[JVN]; found && 0 < len(cont.Title) { + values = append(values, CveContentStr{JVN, cont.Title}) + } + } + + order := CveContentTypes{NVD, NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...) + for _, ctype := range order { + // Only JVN has meaningful title. so return first 100 char of summary + if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Summary) { + summary := strings.Replace(cont.Summary, "\n", " ", -1) + values = append(values, CveContentStr{ + Type: ctype, + Value: summary, + }) + } + } + + for _, adv := range v.DistroAdvisories { + values = append(values, CveContentStr{ + Type: "Vendor", + Value: strings.Replace(adv.Description, "\n", " ", -1), + }) + } + + if len(values) == 0 { + values = []CveContentStr{{ + Type: Unknown, + Value: "-", + }} + } + return +} + +// Summaries returns summaries +func (v VulnInfo) Summaries(lang, myFamily string) (values []CveContentStr) { + if lang == "ja" { + if cont, found := v.CveContents[JVN]; found && 0 < len(cont.Summary) { + summary := cont.Title + summary += "\n" + strings.Replace( + strings.Replace(cont.Summary, "\n", " ", -1), "\r", " ", -1) + values = append(values, CveContentStr{JVN, summary}) + } + } + + order := CveContentTypes{NVD, NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...) + for _, ctype := range order { + if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Summary) { + summary := strings.Replace(cont.Summary, "\n", " ", -1) + values = append(values, CveContentStr{ + Type: ctype, + Value: summary, + }) + } + } + + for _, adv := range v.DistroAdvisories { + values = append(values, CveContentStr{ + Type: "Vendor", + Value: adv.Description, + }) + } + + if len(values) == 0 { + return []CveContentStr{{ + Type: Unknown, + Value: "-", + }} + } + + return +} + +// Cvss2Scores returns CVSS V2 Scores +func (v VulnInfo) Cvss2Scores() (values []CveContentCvss) { + order := []CveContentType{NVD, RedHat, JVN} + for _, ctype := range order { + if cont, found := v.CveContents[ctype]; found && 0 < cont.Cvss2Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + if ctype == NVD { + sev = cvss2ScoreToSeverity(cont.Cvss2Score) + } + values = append(values, CveContentCvss{ + Type: ctype, + Value: Cvss{ + Type: CVSS2, + Score: cont.Cvss2Score, + Vector: cont.Cvss2Vector, + Severity: strings.ToUpper(sev), + }, + }) + } + } + + for _, adv := range v.DistroAdvisories { + if adv.Severity != "" { + values = append(values, CveContentCvss{ + Type: "Vendor", + Value: Cvss{ + Type: CVSS2, + Score: severityToV2ScoreRoughly(adv.Severity), + CalculatedBySeverity: true, + Vector: "-", + Severity: strings.ToUpper(adv.Severity), + }, + }) + } + } + + return +} + +// Cvss3Scores returns CVSS V3 Score +func (v VulnInfo) Cvss3Scores() (values []CveContentCvss) { + // TODO implement NVD + order := []CveContentType{RedHat} + for _, ctype := range order { + if cont, found := v.CveContents[ctype]; found && 0 < cont.Cvss3Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + values = append(values, CveContentCvss{ + Type: ctype, + Value: Cvss{ + Type: CVSS3, + Score: cont.Cvss3Score, + Vector: cont.Cvss3Vector, + Severity: strings.ToUpper(sev), + }, + }) + } + } + return +} + +// MaxCvss3Score returns Max CVSS V3 Score +func (v VulnInfo) MaxCvss3Score() CveContentCvss { + // TODO implement NVD + order := []CveContentType{RedHat} + max := 0.0 + value := CveContentCvss{ + Type: Unknown, + Value: Cvss{Type: CVSS3}, + } + for _, ctype := range order { + if cont, found := v.CveContents[ctype]; found && max < cont.Cvss3Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + value = CveContentCvss{ + Type: ctype, + Value: Cvss{ + Type: CVSS3, + Score: cont.Cvss3Score, + Vector: cont.Cvss3Vector, + Severity: sev, + }, + } + max = cont.Cvss3Score + } + } + return value +} + +// MaxCvssScore returns max CVSS Score +// If there is no CVSS Score, return Severity as a numerical value. +func (v VulnInfo) MaxCvssScore() CveContentCvss { + v3Max := v.MaxCvss3Score() + v2Max := v.MaxCvss2Score() + max := v3Max + if max.Type == Unknown { + return v2Max + } + + if max.Value.Score < v2Max.Value.Score && !v2Max.Value.CalculatedBySeverity { + max = v2Max + } + return max +} + +// MaxCvss2Score returns Max CVSS V2 Score +func (v VulnInfo) MaxCvss2Score() CveContentCvss { + order := []CveContentType{NVD, RedHat, JVN} + max := 0.0 + value := CveContentCvss{ + Type: Unknown, + Value: Cvss{Type: CVSS2}, + } + for _, ctype := range order { + if cont, found := v.CveContents[ctype]; found && max < cont.Cvss2Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + if ctype == NVD { + sev = cvss2ScoreToSeverity(cont.Cvss2Score) + } + value = CveContentCvss{ + Type: ctype, + Value: Cvss{ + Type: CVSS2, + Score: cont.Cvss2Score, + Vector: cont.Cvss2Vector, + Severity: sev, + }, + } + max = cont.Cvss2Score + } + } + if 0 < max { + return value + } + + // If CVSS score isn't on NVD, RedHat and JVN, use OVAL and advisory Severity. + // Convert severity to cvss srore roughly, then returns max severity. + // Only Ubuntu, RedHat and Oracle have severity data in OVAL. + order = []CveContentType{Ubuntu, RedHat, Oracle} + for _, ctype := range order { + if cont, found := v.CveContents[ctype]; found && 0 < len(cont.Severity) { + score := severityToV2ScoreRoughly(cont.Severity) + if max < score { + value = CveContentCvss{ + Type: ctype, + Value: Cvss{ + Type: CVSS2, + Score: score, + CalculatedBySeverity: true, + Vector: cont.Cvss2Vector, + Severity: cont.Severity, + }, + } + } + max = score + } + } + + // Only RedHat, Oracle and Amazon has severity data in advisory. + for _, adv := range v.DistroAdvisories { + if adv.Severity != "" { + score := severityToV2ScoreRoughly(adv.Severity) + if max < score { + value = CveContentCvss{ + Type: "Vendor", + Value: Cvss{ + Type: CVSS2, + Score: score, + CalculatedBySeverity: true, + Vector: "-", + Severity: adv.Severity, + }, + } + } + } + } + return value +} + +// CveContentCvss has CveContentType and Cvss2 +type CveContentCvss struct { + Type CveContentType + Value Cvss +} + +// CvssType Represent the type of CVSS +type CvssType string + +const ( + // CVSS2 means CVSS vesion2 + CVSS2 CvssType = "2" + + // CVSS3 means CVSS vesion3 + CVSS3 CvssType = "3" +) + +// Cvss has CVSS Score +type Cvss struct { + Type CvssType + Score float64 + CalculatedBySeverity bool + Vector string + Severity string +} + +// Format CVSS Score and Vector +func (c Cvss) Format() string { + switch c.Type { + case CVSS2: + return fmt.Sprintf("%3.1f/%s", c.Score, c.Vector) + case CVSS3: + return fmt.Sprintf("%3.1f/CVSS:3.0/%s", c.Score, c.Vector) + } + return "" +} + +func cvss2ScoreToSeverity(score float64) string { + if 7.0 <= score { + return "HIGH" + } else if 4.0 <= score { + return "MEDIUM" + } + return "LOW" +} + +// Amazon Linux Security Advisory +// Critical, Important, Medium, Low +// https://alas.aws.amazon.com/ +// +// RedHat, Oracle OVAL +// Critical, Important, Moderate, Low +// https://access.redhat.com/security/updates/classification +// +// Ubuntu OVAL +// Critical, High, Medium, Low +// https://wiki.ubuntu.com/Bugs/Importance +// https://people.canonical.com/~ubuntu-security/cve/priority.html +func severityToV2ScoreRoughly(severity string) float64 { + switch strings.ToUpper(severity) { + case "CRITICAL": + return 10.0 + case "IMPORTANT", "HIGH": + return 8.9 + case "MODERATE", "MEDIUM": + return 6.9 + case "LOW": + return 3.9 + } + return 0 +} + +// CveContentCvss3 has CveContentType and Cvss3 +// type CveContentCvss3 struct { +// Type CveContentType +// Value Cvss3 +// } + +// Cvss3 has CVSS v3 Score, Vector and Severity +// type Cvss3 struct { +// Score float64 +// Vector string +// Severity string +// } + +// Format CVSS Score and Vector +// func (c Cvss3) Format() string { +// return fmt.Sprintf("%3.1f/CVSS:3.0/%s", c.Score, c.Vector) +// } + +// func cvss3ScoreToSeverity(score float64) string { +// if 9.0 <= score { +// return "CRITICAL" +// } else if 7.0 <= score { +// return "HIGH" +// } else if 4.0 <= score { +// return "MEDIUM" +// } +// return "LOW" +// } + +// FormatMaxCvssScore returns Max CVSS Score +func (v VulnInfo) FormatMaxCvssScore() string { + v2Max := v.MaxCvss2Score() + v3Max := v.MaxCvss3Score() + if v2Max.Value.Score <= v3Max.Value.Score { + return fmt.Sprintf("%3.1f %s (%s)", + v3Max.Value.Score, + strings.ToUpper(v3Max.Value.Severity), + v3Max.Type) + } + return fmt.Sprintf("%3.1f %s (%s)", + v2Max.Value.Score, + strings.ToUpper(v2Max.Value.Severity), + v2Max.Type) +} + +// Cvss2CalcURL returns CVSS v2 caluclator's URL +func (v VulnInfo) Cvss2CalcURL() string { + return "https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=" + v.CveID +} + +// Cvss3CalcURL returns CVSS v3 caluclator's URL +func (v VulnInfo) Cvss3CalcURL() string { + return "https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=" + v.CveID +} + +// VendorLinks returns links of vendor support's URL +func (v VulnInfo) VendorLinks(family string) map[string]string { + links := map[string]string{} + switch family { + case config.RedHat, config.CentOS: + links["RHEL-CVE"] = "https://access.redhat.com/security/cve/" + v.CveID + for _, advisory := range v.DistroAdvisories { + aidURL := strings.Replace(advisory.AdvisoryID, ":", "-", -1) + links[advisory.AdvisoryID] = fmt.Sprintf("https://rhn.redhat.com/errata/%s.html", aidURL) + } + return links + case config.Oracle: + links["Oracle-CVE"] = fmt.Sprintf("https://linux.oracle.com/cve/%s.html", v.CveID) + for _, advisory := range v.DistroAdvisories { + links[advisory.AdvisoryID] = + fmt.Sprintf("https://linux.oracle.com/errata/%s.html", advisory.AdvisoryID) + } + return links + case config.Amazon: + links["RHEL-CVE"] = "https://access.redhat.com/security/cve/" + v.CveID + for _, advisory := range v.DistroAdvisories { + links[advisory.AdvisoryID] = + fmt.Sprintf("https://alas.aws.amazon.com/%s.html", advisory.AdvisoryID) + } + return links + case config.Ubuntu: + links["Ubuntu-CVE"] = "http://people.ubuntu.com/~ubuntu-security/cve/" + v.CveID + return links + case config.Debian: + links["Debian-CVE"] = "https://security-tracker.debian.org/tracker/" + v.CveID + case config.FreeBSD: + for _, advisory := range v.DistroAdvisories { + links["FreeBSD-VuXML"] = fmt.Sprintf("https://vuxml.freebsd.org/freebsd/%s.html", advisory.AdvisoryID) + + } + return links + } + return links +} + +// NilToEmpty set nil slice or map fields to empty to avoid null in JSON +func (v *VulnInfo) NilToEmpty() *VulnInfo { + if v.CpeNames == nil { + v.CpeNames = []string{} + } + if v.DistroAdvisories == nil { + v.DistroAdvisories = []DistroAdvisory{} + } + if v.AffectedPackages == nil { + v.AffectedPackages = PackageStatuses{} + } + if v.CveContents == nil { + v.CveContents = NewCveContents() + } + for key := range v.CveContents { + if v.CveContents[key].Cpes == nil { + cont := v.CveContents[key] + cont.Cpes = []Cpe{} + v.CveContents[key] = cont + } + } + return v +} + +// DistroAdvisory has Amazon Linux, RHEL, FreeBSD Security Advisory information. +type DistroAdvisory struct { + AdvisoryID string + Severity string + Issued time.Time + Updated time.Time + Description string +} + +// Format the distro advisory information +func (p DistroAdvisory) Format() string { + if p.AdvisoryID == "" { + return "" + } + + var delim bytes.Buffer + for i := 0; i < len(p.AdvisoryID); i++ { + delim.WriteString("-") + } + buf := []string{p.AdvisoryID, delim.String(), p.Description} + return strings.Join(buf, "\n") +} + +// Confidence is a ranking how confident the CVE-ID was deteted correctly +// Score: 0 - 100 +type Confidence struct { + Score int + DetectionMethod DetectionMethod +} + +func (c Confidence) String() string { + return fmt.Sprintf("%d / %s", c.Score, c.DetectionMethod) +} + +// DetectionMethod indicates +// - How to detect the CveID +// - How to get the changelog difference between installed and candidate version +type DetectionMethod string + +const ( + // CpeNameMatchStr is a String representation of CpeNameMatch + CpeNameMatchStr = "CpeNameMatch" + + // YumUpdateSecurityMatchStr is a String representation of YumUpdateSecurityMatch + YumUpdateSecurityMatchStr = "YumUpdateSecurityMatch" + + // PkgAuditMatchStr is a String representation of PkgAuditMatch + PkgAuditMatchStr = "PkgAuditMatch" + + // OvalMatchStr is a String representation of OvalMatch + OvalMatchStr = "OvalMatch" + + // ChangelogExactMatchStr is a String representation of ChangelogExactMatch + ChangelogExactMatchStr = "ChangelogExactMatch" + + // ChangelogLenientMatchStr is a String representation of ChangelogLenientMatch + ChangelogLenientMatchStr = "ChangelogLenientMatch" + + // FailedToGetChangelog is a String representation of FailedToGetChangelog + FailedToGetChangelog = "FailedToGetChangelog" + + // FailedToFindVersionInChangelog is a String representation of FailedToFindVersionInChangelog + FailedToFindVersionInChangelog = "FailedToFindVersionInChangelog" +) + +var ( + // CpeNameMatch is a ranking how confident the CVE-ID was deteted correctly + CpeNameMatch = Confidence{100, CpeNameMatchStr} + + // YumUpdateSecurityMatch is a ranking how confident the CVE-ID was deteted correctly + YumUpdateSecurityMatch = Confidence{100, YumUpdateSecurityMatchStr} + + // PkgAuditMatch is a ranking how confident the CVE-ID was deteted correctly + PkgAuditMatch = Confidence{100, PkgAuditMatchStr} + + // OvalMatch is a ranking how confident the CVE-ID was deteted correctly + OvalMatch = Confidence{100, OvalMatchStr} + + // ChangelogExactMatch is a ranking how confident the CVE-ID was deteted correctly + ChangelogExactMatch = Confidence{95, ChangelogExactMatchStr} + + // ChangelogLenientMatch is a ranking how confident the CVE-ID was deteted correctly + ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr} +) diff --git a/models/vulninfos_test.go b/models/vulninfos_test.go new file mode 100644 index 00000000..93adaddb --- /dev/null +++ b/models/vulninfos_test.go @@ -0,0 +1,936 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models + +import ( + "reflect" + "testing" +) + +func TestTitles(t *testing.T) { + type in struct { + lang string + cont VulnInfo + } + var tests = []struct { + in in + out []CveContentStr + }{ + // lang: ja + { + in: in{ + lang: "ja", + cont: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Title: "Title1", + }, + RedHat: { + Type: RedHat, + Summary: "Summary RedHat", + }, + NVD: { + Type: NVD, + Summary: "Summary NVD", + // Severity is NIOT included in NVD + }, + }, + }, + }, + out: []CveContentStr{ + { + Type: JVN, + Value: "Title1", + }, + { + Type: NVD, + Value: "Summary NVD", + }, + { + Type: RedHat, + Value: "Summary RedHat", + }, + }, + }, + // lang: en + { + in: in{ + lang: "en", + cont: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Title: "Title1", + }, + RedHat: { + Type: RedHat, + Summary: "Summary RedHat", + }, + NVD: { + Type: NVD, + Summary: "Summary NVD", + // Severity is NIOT included in NVD + }, + }, + }, + }, + out: []CveContentStr{ + { + Type: NVD, + Value: "Summary NVD", + }, + { + Type: RedHat, + Value: "Summary RedHat", + }, + }, + }, + // lang: empty + { + in: in{ + lang: "en", + cont: VulnInfo{}, + }, + out: []CveContentStr{ + { + Type: Unknown, + Value: "-", + }, + }, + }, + } + for _, tt := range tests { + actual := tt.in.cont.Titles(tt.in.lang, "redhat") + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestSummaries(t *testing.T) { + type in struct { + lang string + cont VulnInfo + } + var tests = []struct { + in in + out []CveContentStr + }{ + // lang: ja + { + in: in{ + lang: "ja", + cont: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Title: "Title JVN", + Summary: "Summary JVN", + }, + RedHat: { + Type: RedHat, + Summary: "Summary RedHat", + }, + NVD: { + Type: NVD, + Summary: "Summary NVD", + // Severity is NIOT included in NVD + }, + }, + }, + }, + out: []CveContentStr{ + { + Type: JVN, + Value: "Title JVN\nSummary JVN", + }, + { + Type: NVD, + Value: "Summary NVD", + }, + { + Type: RedHat, + Value: "Summary RedHat", + }, + }, + }, + // lang: en + { + in: in{ + lang: "en", + cont: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Title: "Title JVN", + Summary: "Summary JVN", + }, + RedHat: { + Type: RedHat, + Summary: "Summary RedHat", + }, + NVD: { + Type: NVD, + Summary: "Summary NVD", + // Severity is NIOT included in NVD + }, + }, + }, + }, + out: []CveContentStr{ + { + Type: NVD, + Value: "Summary NVD", + }, + { + Type: RedHat, + Value: "Summary RedHat", + }, + }, + }, + // lang: empty + { + in: in{ + lang: "en", + cont: VulnInfo{}, + }, + out: []CveContentStr{ + { + Type: Unknown, + Value: "-", + }, + }, + }, + } + for _, tt := range tests { + actual := tt.in.cont.Summaries(tt.in.lang, "redhat") + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestCountGroupBySeverity(t *testing.T) { + var tests = []struct { + in VulnInfos + out map[string]int + }{ + { + in: VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 6.0, + }, + RedHat: { + Type: RedHat, + Cvss2Score: 7.0, + }, + }, + }, + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 2.0, + }, + }, + }, + "CVE-2017-0004": { + CveID: "CVE-2017-0004", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 5.0, + }, + }, + }, + "CVE-2017-0005": { + CveID: "CVE-2017-0005", + }, + }, + out: map[string]int{ + "High": 1, + "Medium": 1, + "Low": 1, + "Unknown": 1, + }, + }, + } + for _, tt := range tests { + actual := tt.in.CountGroupBySeverity() + for k := range tt.out { + if tt.out[k] != actual[k] { + t.Errorf("\nexpected %s: %d\n actual %d\n", + k, tt.out[k], actual[k]) + } + } + } +} + +func TestToSortedSlice(t *testing.T) { + var tests = []struct { + in VulnInfos + out []VulnInfo + }{ + { + in: VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 6.0, + }, + RedHat: { + Type: RedHat, + Cvss3Score: 7.0, + }, + }, + }, + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 7.0, + }, + RedHat: { + Type: RedHat, + Cvss3Score: 8.0, + }, + }, + }, + }, + out: []VulnInfo{ + { + CveID: "CVE-2017-0001", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 7.0, + }, + RedHat: { + Type: RedHat, + Cvss3Score: 8.0, + }, + }, + }, + { + CveID: "CVE-2017-0002", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 6.0, + }, + RedHat: { + Type: RedHat, + Cvss3Score: 7.0, + }, + }, + }, + }, + }, + // When max scores are the same, sort by CVE-ID + { + in: VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 6.0, + }, + RedHat: { + Type: RedHat, + Cvss3Score: 7.0, + }, + }, + }, + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: CveContents{ + RedHat: { + Type: RedHat, + Cvss2Score: 7.0, + }, + }, + }, + }, + out: []VulnInfo{ + { + CveID: "CVE-2017-0001", + CveContents: CveContents{ + RedHat: { + Type: RedHat, + Cvss2Score: 7.0, + }, + }, + }, + { + CveID: "CVE-2017-0002", + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss2Score: 6.0, + }, + RedHat: { + Type: RedHat, + Cvss3Score: 7.0, + }, + }, + }, + }, + }, + // When there are no cvss scores, sort by severity + { + in: VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "High", + }, + }, + }, + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "Low", + }, + }, + }, + }, + out: []VulnInfo{ + { + CveID: "CVE-2017-0002", + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "High", + }, + }, + }, + { + CveID: "CVE-2017-0001", + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "Low", + }, + }, + }, + }, + }, + } + for _, tt := range tests { + actual := tt.in.ToSortedSlice() + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestCvss2Scores(t *testing.T) { + var tests = []struct { + in VulnInfo + out []CveContentCvss + }{ + { + in: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Severity: "HIGH", + Cvss2Score: 8.2, + Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + }, + RedHat: { + Type: RedHat, + Severity: "HIGH", + Cvss2Score: 8.0, + Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + }, + NVD: { + Type: NVD, + Cvss2Score: 8.1, + Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + // Severity is NIOT included in NVD + }, + }, + }, + out: []CveContentCvss{ + { + Type: NVD, + Value: Cvss{ + Type: CVSS2, + Score: 8.1, + Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + Severity: "HIGH", + }, + }, + { + Type: RedHat, + Value: Cvss{ + Type: CVSS2, + Score: 8.0, + Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + Severity: "HIGH", + }, + }, + { + Type: JVN, + Value: Cvss{ + Type: CVSS2, + Score: 8.2, + Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + Severity: "HIGH", + }, + }, + }, + }, + // Empty + { + in: VulnInfo{}, + out: nil, + }, + } + for i, tt := range tests { + actual := tt.in.Cvss2Scores() + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("[%d] expected: %v\n actual: %v\n", i, tt.out, actual) + } + } +} + +func TestMaxCvss2Scores(t *testing.T) { + var tests = []struct { + in VulnInfo + out CveContentCvss + }{ + { + in: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Severity: "HIGH", + Cvss2Score: 8.2, + Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + }, + RedHat: { + Type: RedHat, + Severity: "HIGH", + Cvss2Score: 8.0, + Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + }, + NVD: { + Type: NVD, + Cvss2Score: 8.1, + Cvss2Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + // Severity is NIOT included in NVD + }, + }, + }, + out: CveContentCvss{ + Type: JVN, + Value: Cvss{ + Type: CVSS2, + Score: 8.2, + Vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + Severity: "HIGH", + }, + }, + }, + // Severity in OVAL + { + in: VulnInfo{ + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "HIGH", + }, + }, + }, + out: CveContentCvss{ + Type: Ubuntu, + Value: Cvss{ + Type: CVSS2, + Score: 8.9, + CalculatedBySeverity: true, + Severity: "HIGH", + }, + }, + }, + // Empty + { + in: VulnInfo{}, + out: CveContentCvss{ + Type: Unknown, + Value: Cvss{ + Type: CVSS2, + Score: 0.0, + Vector: "", + Severity: "", + }, + }, + }, + } + for i, tt := range tests { + actual := tt.in.MaxCvss2Score() + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("[%d] expected: %v\n actual: %v\n", i, tt.out, actual) + } + } +} + +func TestCvss3Scores(t *testing.T) { + var tests = []struct { + in VulnInfo + out []CveContentCvss + }{ + { + in: VulnInfo{ + CveContents: CveContents{ + RedHat: { + Type: RedHat, + Severity: "HIGH", + Cvss3Score: 8.0, + Cvss3Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L", + }, + NVD: { + Type: NVD, + Cvss3Score: 8.1, + Cvss3Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L", + // Severity is NIOT included in NVD + }, + }, + }, + out: []CveContentCvss{ + { + Type: RedHat, + Value: Cvss{ + Type: CVSS3, + Score: 8.0, + Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L", + Severity: "HIGH", + }, + }, + }, + }, + // Empty + { + in: VulnInfo{}, + out: nil, + }, + } + for _, tt := range tests { + actual := tt.in.Cvss3Scores() + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestMaxCvss3Scores(t *testing.T) { + var tests = []struct { + in VulnInfo + out CveContentCvss + }{ + { + in: VulnInfo{ + CveContents: CveContents{ + RedHat: { + Type: RedHat, + Severity: "HIGH", + Cvss3Score: 8.0, + Cvss3Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L", + }, + }, + }, + out: CveContentCvss{ + Type: RedHat, + Value: Cvss{ + Type: CVSS3, + Score: 8.0, + Vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L", + Severity: "HIGH", + }, + }, + }, + // Empty + { + in: VulnInfo{}, + out: CveContentCvss{ + Type: Unknown, + Value: Cvss{ + Type: CVSS3, + Score: 0.0, + Vector: "", + Severity: "", + }, + }, + }, + } + for _, tt := range tests { + actual := tt.in.MaxCvss3Score() + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestMaxCvssScores(t *testing.T) { + var tests = []struct { + in VulnInfo + out CveContentCvss + }{ + { + in: VulnInfo{ + CveContents: CveContents{ + NVD: { + Type: NVD, + Cvss3Score: 7.0, + }, + RedHat: { + Type: RedHat, + Cvss2Score: 8.0, + }, + }, + }, + out: CveContentCvss{ + Type: RedHat, + Value: Cvss{ + Type: CVSS2, + Score: 8.0, + }, + }, + }, + { + in: VulnInfo{ + CveContents: CveContents{ + RedHat: { + Type: RedHat, + Cvss3Score: 8.0, + }, + }, + }, + out: CveContentCvss{ + Type: RedHat, + Value: Cvss{ + Type: CVSS3, + Score: 8.0, + }, + }, + }, + //2 + { + in: VulnInfo{ + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "HIGH", + }, + }, + }, + out: CveContentCvss{ + Type: Ubuntu, + Value: Cvss{ + Type: CVSS2, + Score: 8.9, + CalculatedBySeverity: true, + Severity: "HIGH", + }, + }, + }, + //3 + { + in: VulnInfo{ + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "MEDIUM", + }, + NVD: { + Type: NVD, + Cvss2Score: 7.0, + }, + }, + }, + out: CveContentCvss{ + Type: NVD, + Value: Cvss{ + Type: CVSS2, + Score: 7.0, + Severity: "HIGH", + }, + }, + }, + //4 + { + in: VulnInfo{ + DistroAdvisories: []DistroAdvisory{ + { + Severity: "HIGH", + }, + }, + }, + out: CveContentCvss{ + Type: "Vendor", + Value: Cvss{ + Type: CVSS2, + Score: 8.9, + CalculatedBySeverity: true, + Vector: "-", + Severity: "HIGH", + }, + }, + }, + { + in: VulnInfo{ + CveContents: CveContents{ + Ubuntu: { + Type: Ubuntu, + Severity: "MEDIUM", + }, + NVD: { + Type: NVD, + Cvss2Score: 4.0, + }, + }, + DistroAdvisories: []DistroAdvisory{ + { + Severity: "HIGH", + }, + }, + }, + out: CveContentCvss{ + Type: NVD, + Value: Cvss{ + Type: CVSS2, + Score: 4, + Severity: "MEDIUM", + }, + }, + }, + // Empty + { + in: VulnInfo{}, + out: CveContentCvss{ + Type: Unknown, + Value: Cvss{ + Type: CVSS2, + Score: 0, + }, + }, + }, + } + for i, tt := range tests { + actual := tt.in.MaxCvssScore() + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\n[%d] expected: %v\n actual: %v\n", i, tt.out, actual) + } + } +} + +func TestFormatMaxCvssScore(t *testing.T) { + var tests = []struct { + in VulnInfo + out string + }{ + { + in: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Severity: "HIGH", + Cvss2Score: 8.3, + }, + RedHat: { + Type: RedHat, + Severity: "HIGH", + Cvss3Score: 8.0, + }, + NVD: { + Type: NVD, + Cvss2Score: 8.1, + // Severity is NIOT included in NVD + }, + }, + }, + out: "8.3 HIGH (jvn)", + }, + { + in: VulnInfo{ + CveContents: CveContents{ + JVN: { + Type: JVN, + Severity: "HIGH", + Cvss2Score: 8.3, + }, + RedHat: { + Type: RedHat, + Severity: "HIGH", + Cvss2Score: 8.0, + Cvss3Score: 9.9, + }, + NVD: { + Type: NVD, + Cvss2Score: 8.1, + }, + }, + }, + out: "9.9 HIGH (redhat)", + }, + } + for _, tt := range tests { + actual := tt.in.FormatMaxCvssScore() + if !reflect.DeepEqual(tt.out, actual) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, actual) + } + } +} + +func TestSortPackageStatues(t *testing.T) { + var tests = []struct { + in PackageStatuses + out PackageStatuses + }{ + { + in: PackageStatuses{ + {Name: "b"}, + {Name: "a"}, + }, + out: PackageStatuses{ + {Name: "a"}, + {Name: "b"}, + }, + }, + } + for _, tt := range tests { + tt.in.Sort() + if !reflect.DeepEqual(tt.in, tt.out) { + t.Errorf("\nexpected: %v\n actual: %v\n", tt.out, tt.in) + } + } +} diff --git a/oval/debian.go b/oval/debian.go new file mode 100644 index 00000000..873a824d --- /dev/null +++ b/oval/debian.go @@ -0,0 +1,266 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package oval + +import ( + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + ovalmodels "github.com/kotakanbe/goval-dictionary/models" +) + +// DebianBase is the base struct of Debian and Ubuntu +type DebianBase struct { + Base +} + +func (o DebianBase) update(r *models.ScanResult, defPacks defPacks) { + ovalContent := *o.convertToModel(&defPacks.def) + ovalContent.Type = models.NewCveContentType(o.family) + vinfo, ok := r.ScannedCves[defPacks.def.Debian.CveID] + if !ok { + util.Log.Debugf("%s is newly detected by OVAL", defPacks.def.Debian.CveID) + vinfo = models.VulnInfo{ + CveID: defPacks.def.Debian.CveID, + Confidence: models.OvalMatch, + CveContents: models.NewCveContents(ovalContent), + } + } else { + cveContents := vinfo.CveContents + ctype := models.NewCveContentType(o.family) + if _, ok := vinfo.CveContents[ctype]; ok { + util.Log.Debugf("%s OVAL will be overwritten", + defPacks.def.Debian.CveID) + } else { + util.Log.Debugf("%s is also detected by OVAL", + defPacks.def.Debian.CveID) + cveContents = models.CveContents{} + } + if vinfo.Confidence.Score < models.OvalMatch.Score { + vinfo.Confidence = models.OvalMatch + } + cveContents[ctype] = ovalContent + vinfo.CveContents = cveContents + } + + // uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames) + for _, pack := range vinfo.AffectedPackages { + defPacks.actuallyAffectedPackNames[pack.Name] = true + } + vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family, r.Packages) + vinfo.AffectedPackages.Sort() + r.ScannedCves[defPacks.def.Debian.CveID] = vinfo +} + +func (o DebianBase) convertToModel(def *ovalmodels.Definition) *models.CveContent { + var refs []models.Reference + for _, r := range def.References { + refs = append(refs, models.Reference{ + Link: r.RefURL, + Source: r.Source, + RefID: r.RefID, + }) + } + + return &models.CveContent{ + CveID: def.Debian.CveID, + Title: def.Title, + Summary: def.Description, + Severity: def.Advisory.Severity, + References: refs, + } +} + +// Debian is the interface for Debian OVAL +type Debian struct { + DebianBase +} + +// NewDebian creates OVAL client for Debian +func NewDebian() Debian { + return Debian{ + DebianBase{ + Base{ + family: config.Debian, + }, + }, + } +} + +// FillWithOval returns scan result after updating CVE info by OVAL +func (o Debian) FillWithOval(r *models.ScanResult) (err error) { + + //Debian's uname gives both of kernel release(uname -r), version(kernel-image version) + linuxImage := "linux-image-" + r.RunningKernel.Release + // Add linux and set the version of running kernel to search OVAL. + if r.Container.ContainerID == "" { + r.Packages["linux"] = models.Package{ + Name: "linux", + Version: r.RunningKernel.Version, + } + } + + var relatedDefs ovalResult + if o.isFetchViaHTTP() { + if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil { + return err + } + } else { + if relatedDefs, err = getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages); err != nil { + return err + } + } + + delete(r.Packages, "linux") + + for _, defPacks := range relatedDefs.entries { + // Remove linux added above to search for oval + // linux is not a real package name (key of affected packages in OVAL) + if _, ok := defPacks.actuallyAffectedPackNames["linux"]; ok { + defPacks.actuallyAffectedPackNames[linuxImage] = true + delete(defPacks.actuallyAffectedPackNames, "linux") + for i, p := range defPacks.def.AffectedPacks { + if p.Name == "linux" { + p.Name = linuxImage + defPacks.def.AffectedPacks[i] = p + } + } + } + o.update(r, defPacks) + } + + for _, vuln := range r.ScannedCves { + if cont, ok := vuln.CveContents[models.Debian]; ok { + cont.SourceLink = "https://security-tracker.debian.org/tracker/" + cont.CveID + vuln.CveContents[models.Debian] = cont + } + } + return nil +} + +// Ubuntu is the interface for Debian OVAL +type Ubuntu struct { + DebianBase +} + +// NewUbuntu creates OVAL client for Debian +func NewUbuntu() Ubuntu { + return Ubuntu{ + DebianBase{ + Base{ + family: config.Ubuntu, + }, + }, + } +} + +// FillWithOval returns scan result after updating CVE info by OVAL +func (o Ubuntu) FillWithOval(r *models.ScanResult) (err error) { + ovalKernelImageNames := []string{ + "linux-aws", + "linux-azure", + "linux-flo", + "linux-gcp", + "linux-gke", + "linux-goldfish", + "linux-hwe", + "linux-hwe-edge", + "linux-kvm", + "linux-mako", + "linux-raspi2", + "linux-snapdragon", + } + linuxImage := "linux-image-" + r.RunningKernel.Release + + found := false + if r.Container.ContainerID == "" { + for _, n := range ovalKernelImageNames { + if _, ok := r.Packages[n]; ok { + v, ok := r.Packages[linuxImage] + if ok { + // Set running kernel version + p := r.Packages[n] + p.Version = v.Version + p.NewVersion = v.NewVersion + r.Packages[n] = p + } else { + util.Log.Warnf("Running kernel image %s is not found: %s", + linuxImage, r.RunningKernel.Version) + } + found = true + break + } + } + + if !found { + // linux-generic is described as "linux" in Ubuntu's oval. + // Add "linux" and set the version of running kernel to search OVAL. + v, ok := r.Packages[linuxImage] + if ok { + r.Packages["linux"] = models.Package{ + Name: "linux", + Version: v.Version, + NewVersion: v.NewVersion, + } + } else { + util.Log.Warnf("%s is not found. Running: %s", + linuxImage, r.RunningKernel.Release) + } + } + } + + var relatedDefs ovalResult + if o.isFetchViaHTTP() { + if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil { + return err + } + } else { + if relatedDefs, err = getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages); err != nil { + return err + } + } + + if !found { + delete(r.Packages, "linux") + } + + for _, defPacks := range relatedDefs.entries { + + // Remove "linux" added above to search for oval + // "linux" is not a real package name (key of affected packages in OVAL) + if _, ok := defPacks.actuallyAffectedPackNames["linux"]; !found && ok { + defPacks.actuallyAffectedPackNames[linuxImage] = true + delete(defPacks.actuallyAffectedPackNames, "linux") + for i, p := range defPacks.def.AffectedPacks { + if p.Name == "linux" { + p.Name = linuxImage + defPacks.def.AffectedPacks[i] = p + } + } + } + o.update(r, defPacks) + } + + for _, vuln := range r.ScannedCves { + if cont, ok := vuln.CveContents[models.Ubuntu]; ok { + cont.SourceLink = "http://people.ubuntu.com/~ubuntu-security/cve/" + cont.CveID + vuln.CveContents[models.Ubuntu] = cont + } + } + return nil +} diff --git a/oval/debian_test.go b/oval/debian_test.go new file mode 100644 index 00000000..f522d1c6 --- /dev/null +++ b/oval/debian_test.go @@ -0,0 +1,75 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package oval + +import ( + "reflect" + "testing" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + ovalmodels "github.com/kotakanbe/goval-dictionary/models" +) + +func TestPackNamesOfUpdateDebian(t *testing.T) { + var tests = []struct { + in models.ScanResult + defPacks defPacks + out models.ScanResult + }{ + { + in: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2000-1000": models.VulnInfo{ + AffectedPackages: models.PackageStatuses{{Name: "packA"}}, + }, + }, + }, + defPacks: defPacks{ + def: ovalmodels.Definition{ + Debian: ovalmodels.Debian{ + CveID: "CVE-2000-1000", + }, + }, + actuallyAffectedPackNames: map[string]bool{ + "packB": true, + }, + }, + out: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2000-1000": models.VulnInfo{ + AffectedPackages: models.PackageStatuses{ + {Name: "packA"}, + {Name: "packB"}, + }, + }, + }, + }, + }, + } + + util.Log = util.NewCustomLogger(config.ServerInfo{}) + for i, tt := range tests { + Debian{}.update(&tt.in, tt.defPacks) + e := tt.out.ScannedCves["CVE-2000-1000"].AffectedPackages + a := tt.in.ScannedCves["CVE-2000-1000"].AffectedPackages + if !reflect.DeepEqual(a, e) { + t.Errorf("[%d] expected: %v\n actual: %v\n", i, e, a) + } + } +} diff --git a/oval/oval.go b/oval/oval.go new file mode 100644 index 00000000..6fb3846b --- /dev/null +++ b/oval/oval.go @@ -0,0 +1,150 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package oval + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + "github.com/kotakanbe/goval-dictionary/db" + ovallog "github.com/kotakanbe/goval-dictionary/log" + "github.com/parnurzeal/gorequest" +) + +// Client is the interface of OVAL client. +type Client interface { + CheckHTTPHealth() error + FillWithOval(r *models.ScanResult) error + + // CheckIfOvalFetched checks if oval entries are in DB by family, release. + CheckIfOvalFetched(string, string) (bool, error) + CheckIfOvalFresh(string, string) (bool, error) +} + +// Base is a base struct +type Base struct { + family string +} + +// CheckHTTPHealth do health check +func (b Base) CheckHTTPHealth() error { + if !b.isFetchViaHTTP() { + return nil + } + + url := fmt.Sprintf("%s/health", config.Conf.OvalDBURL) + var errs []error + var resp *http.Response + resp, _, errs = gorequest.New().Get(url).End() + // resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() + // resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + return fmt.Errorf("Failed to request to OVAL server. url: %s, errs: %v", + url, errs) + } + return nil +} + +// CheckIfOvalFetched checks if oval entries are in DB by family, release. +func (b Base) CheckIfOvalFetched(osFamily, release string) (fetched bool, err error) { + ovallog.Initialize(config.Conf.LogDir) + if !b.isFetchViaHTTP() { + var ovaldb db.DB + if ovaldb, err = db.NewDB( + osFamily, + config.Conf.OvalDBType, + config.Conf.OvalDBPath, + config.Conf.DebugSQL, + ); err != nil { + return false, err + } + defer ovaldb.CloseDB() + count, err := ovaldb.CountDefs(osFamily, release) + if err != nil { + return false, fmt.Errorf("Failed to count OVAL defs: %s, %s, %v", + osFamily, release, err) + } + return 0 < count, nil + } + + url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "count", osFamily, release) + resp, body, errs := gorequest.New().Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", + errs, url, resp) + } + count := 0 + if err := json.Unmarshal([]byte(body), &count); err != nil { + return false, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", + body, err) + } + return 0 < count, nil +} + +// CheckIfOvalFresh checks if oval entries are fresh enough +func (b Base) CheckIfOvalFresh(osFamily, release string) (ok bool, err error) { + ovallog.Initialize(config.Conf.LogDir) + var lastModified time.Time + if !b.isFetchViaHTTP() { + var ovaldb db.DB + if ovaldb, err = db.NewDB( + osFamily, + config.Conf.OvalDBType, + config.Conf.OvalDBPath, + config.Conf.DebugSQL, + ); err != nil { + return false, err + } + defer ovaldb.CloseDB() + lastModified = ovaldb.GetLastModified(osFamily, release) + } else { + url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "lastmodified", osFamily, release) + resp, body, errs := gorequest.New().Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", + errs, url, resp) + } + + if err := json.Unmarshal([]byte(body), &lastModified); err != nil { + return false, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", + body, err) + } + } + + major := strings.Split(release, ".")[0] + since := time.Now() + since = since.AddDate(0, 0, -3) + if lastModified.Before(since) { + util.Log.Warnf("OVAL for %s %s is old, last modified is %s. It's recommended to update OVAL to improve scanning accuracy. How to update OVAL database, see https://github.com/kotakanbe/goval-dictionary#usage", + osFamily, major, lastModified) + return false, nil + } + util.Log.Infof("OVAL is fresh: %s %s ", osFamily, major) + return true, nil +} + +func (b Base) isFetchViaHTTP() bool { + // Default value of OvalDBType is sqlite3 + return config.Conf.OvalDBURL != "" && config.Conf.OvalDBType == "sqlite3" +} diff --git a/oval/redhat.go b/oval/redhat.go new file mode 100644 index 00000000..cd5c28f5 --- /dev/null +++ b/oval/redhat.go @@ -0,0 +1,224 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package oval + +import ( + "fmt" + "strconv" + "strings" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + ovalmodels "github.com/kotakanbe/goval-dictionary/models" +) + +// RedHatBase is the base struct for RedHat and CentOS +type RedHatBase struct { + Base +} + +// FillWithOval returns scan result after updating CVE info by OVAL +func (o RedHatBase) FillWithOval(r *models.ScanResult) (err error) { + var relatedDefs ovalResult + if o.isFetchViaHTTP() { + if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil { + return err + } + } else { + if relatedDefs, err = getDefsByPackNameFromOvalDB( + o.family, r.Release, r.Packages); err != nil { + return err + } + } + + for _, defPacks := range relatedDefs.entries { + o.update(r, defPacks) + } + + for _, vuln := range r.ScannedCves { + switch models.NewCveContentType(o.family) { + case models.RedHat: + if cont, ok := vuln.CveContents[models.RedHat]; ok { + cont.SourceLink = "https://access.redhat.com/security/cve/" + cont.CveID + vuln.CveContents[models.RedHat] = cont + } + case models.Oracle: + if cont, ok := vuln.CveContents[models.Oracle]; ok { + cont.SourceLink = fmt.Sprintf("https://linux.oracle.com/cve/%s.html", cont.CveID) + vuln.CveContents[models.Oracle] = cont + } + } + } + return nil +} + +func (o RedHatBase) update(r *models.ScanResult, defPacks defPacks) { + ctype := models.NewCveContentType(o.family) + for _, cve := range defPacks.def.Advisory.Cves { + ovalContent := *o.convertToModel(cve.CveID, &defPacks.def) + vinfo, ok := r.ScannedCves[cve.CveID] + if !ok { + util.Log.Debugf("%s is newly detected by OVAL", cve.CveID) + vinfo = models.VulnInfo{ + CveID: cve.CveID, + Confidence: models.OvalMatch, + CveContents: models.NewCveContents(ovalContent), + } + } else { + cveContents := vinfo.CveContents + if _, ok := vinfo.CveContents[ctype]; ok { + util.Log.Debugf("%s OVAL will be overwritten", cve.CveID) + } else { + util.Log.Debugf("%s also detected by OVAL", cve.CveID) + cveContents = models.CveContents{} + } + + if vinfo.Confidence.Score < models.OvalMatch.Score { + vinfo.Confidence = models.OvalMatch + } + cveContents[ctype] = ovalContent + vinfo.CveContents = cveContents + } + + // uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames) + for _, pack := range vinfo.AffectedPackages { + defPacks.actuallyAffectedPackNames[pack.Name] = true + } + vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family, r.Packages) + vinfo.AffectedPackages.Sort() + r.ScannedCves[cve.CveID] = vinfo + } +} + +func (o RedHatBase) convertToModel(cveID string, def *ovalmodels.Definition) *models.CveContent { + for _, cve := range def.Advisory.Cves { + if cve.CveID != cveID { + continue + } + var refs []models.Reference + for _, r := range def.References { + refs = append(refs, models.Reference{ + Link: r.RefURL, + Source: r.Source, + RefID: r.RefID, + }) + } + + score2, vec2 := o.parseCvss2(cve.Cvss2) + score3, vec3 := o.parseCvss3(cve.Cvss3) + + severity := def.Advisory.Severity + if cve.Impact != "" { + severity = cve.Impact + } + + return &models.CveContent{ + Type: models.NewCveContentType(o.family), + CveID: cve.CveID, + Title: def.Title, + Summary: def.Description, + Severity: severity, + Cvss2Score: score2, + Cvss2Vector: vec2, + Cvss3Score: score3, + Cvss3Vector: vec3, + References: refs, + CweID: cve.Cwe, + Published: def.Advisory.Issued, + LastModified: def.Advisory.Updated, + } + } + return nil +} + +// ParseCvss2 divide CVSSv2 string into score and vector +// 5/AV:N/AC:L/Au:N/C:N/I:N/A:P +func (o RedHatBase) parseCvss2(scoreVector string) (score float64, vector string) { + var err error + ss := strings.Split(scoreVector, "/") + if 1 < len(ss) { + if score, err = strconv.ParseFloat(ss[0], 64); err != nil { + return 0, "" + } + return score, strings.Join(ss[1:len(ss)], "/") + } + return 0, "" +} + +// ParseCvss3 divide CVSSv3 string into score and vector +// 5.6/CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L +func (o RedHatBase) parseCvss3(scoreVector string) (score float64, vector string) { + var err error + ss := strings.Split(scoreVector, "/CVSS:3.0/") + if 1 < len(ss) { + if score, err = strconv.ParseFloat(ss[0], 64); err != nil { + return 0, "" + } + return score, strings.Join(ss[1:len(ss)], "/") + } + return 0, "" +} + +// RedHat is the interface for RedhatBase OVAL +type RedHat struct { + RedHatBase +} + +// NewRedhat creates OVAL client for Redhat +func NewRedhat() RedHat { + return RedHat{ + RedHatBase{ + Base{ + family: config.RedHat, + }, + }, + } +} + +// CentOS is the interface for CentOS OVAL +type CentOS struct { + RedHatBase +} + +// NewCentOS creates OVAL client for CentOS +func NewCentOS() CentOS { + return CentOS{ + RedHatBase{ + Base{ + family: config.CentOS, + }, + }, + } +} + +// Oracle is the interface for CentOS OVAL +type Oracle struct { + RedHatBase +} + +// NewOracle creates OVAL client for Oracle +func NewOracle() Oracle { + return Oracle{ + RedHatBase{ + Base{ + family: config.Oracle, + }, + }, + } +} diff --git a/oval/redhat_test.go b/oval/redhat_test.go new file mode 100644 index 00000000..7914686c --- /dev/null +++ b/oval/redhat_test.go @@ -0,0 +1,145 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package oval + +import ( + "reflect" + "testing" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + ovalmodels "github.com/kotakanbe/goval-dictionary/models" +) + +func TestParseCvss2(t *testing.T) { + type out struct { + score float64 + vector string + } + var tests = []struct { + in string + out out + }{ + { + in: "5/AV:N/AC:L/Au:N/C:N/I:N/A:P", + out: out{ + score: 5.0, + vector: "AV:N/AC:L/Au:N/C:N/I:N/A:P", + }, + }, + { + in: "", + out: out{ + score: 0, + vector: "", + }, + }, + } + for _, tt := range tests { + s, v := RedHatBase{}.parseCvss2(tt.in) + if s != tt.out.score || v != tt.out.vector { + t.Errorf("\nexpected: %f, %s\n actual: %f, %s", + tt.out.score, tt.out.vector, s, v) + } + } +} + +func TestParseCvss3(t *testing.T) { + type out struct { + score float64 + vector string + } + var tests = []struct { + in string + out out + }{ + { + in: "5.6/CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L", + out: out{ + score: 5.6, + vector: "AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:L/A:L", + }, + }, + { + in: "", + out: out{ + score: 0, + vector: "", + }, + }, + } + for _, tt := range tests { + s, v := RedHatBase{}.parseCvss3(tt.in) + if s != tt.out.score || v != tt.out.vector { + t.Errorf("\nexpected: %f, %s\n actual: %f, %s", + tt.out.score, tt.out.vector, s, v) + } + } +} + +func TestPackNamesOfUpdate(t *testing.T) { + var tests = []struct { + in models.ScanResult + defPacks defPacks + out models.ScanResult + }{ + { + in: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2000-1000": models.VulnInfo{ + AffectedPackages: models.PackageStatuses{{Name: "packA"}}, + }, + }, + }, + defPacks: defPacks{ + def: ovalmodels.Definition{ + Advisory: ovalmodels.Advisory{ + Cves: []ovalmodels.Cve{ + { + CveID: "CVE-2000-1000", + }, + }, + }, + }, + actuallyAffectedPackNames: map[string]bool{ + "packB": true, + }, + }, + out: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2000-1000": models.VulnInfo{ + AffectedPackages: models.PackageStatuses{ + {Name: "packA"}, + {Name: "packB"}, + }, + }, + }, + }, + }, + } + + util.Log = util.NewCustomLogger(config.ServerInfo{}) + for i, tt := range tests { + RedHat{}.update(&tt.in, tt.defPacks) + e := tt.out.ScannedCves["CVE-2000-1000"].AffectedPackages + a := tt.in.ScannedCves["CVE-2000-1000"].AffectedPackages + if !reflect.DeepEqual(a, e) { + t.Errorf("[%d] expected: %v\n actual: %v\n", i, e, a) + } + } +} diff --git a/oval/util.go b/oval/util.go new file mode 100644 index 00000000..e6f9542d --- /dev/null +++ b/oval/util.go @@ -0,0 +1,332 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package oval + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/cenkalti/backoff" + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + debver "github.com/knqyf263/go-deb-version" + rpmver "github.com/knqyf263/go-rpm-version" + "github.com/kotakanbe/goval-dictionary/db" + ovallog "github.com/kotakanbe/goval-dictionary/log" + ovalmodels "github.com/kotakanbe/goval-dictionary/models" + "github.com/parnurzeal/gorequest" +) + +type ovalResult struct { + entries []defPacks +} + +type defPacks struct { + def ovalmodels.Definition + actuallyAffectedPackNames map[string]bool +} + +func (e defPacks) toPackStatuses(family string, packs models.Packages) (ps models.PackageStatuses) { + switch family { + case config.Ubuntu: + packNotFixedYet := map[string]bool{} + for _, p := range e.def.AffectedPacks { + packNotFixedYet[p.Name] = p.NotFixedYet + } + for k := range e.actuallyAffectedPackNames { + ps = append(ps, models.PackageStatus{ + Name: k, + NotFixedYet: packNotFixedYet[k], + }) + } + + case config.CentOS, config.Debian: + // There are many packages that has been fixed in RedHat, but not been fixed in CentOS + for name := range e.actuallyAffectedPackNames { + pack, ok := packs[name] + if !ok { + util.Log.Warnf("Faild to find in Package list: %s", name) + return + } + + ovalPackVer := "" + for _, p := range e.def.AffectedPacks { + if p.Name == name { + ovalPackVer = p.Version + break + } + } + if ovalPackVer == "" { + util.Log.Warnf("Faild to find in Oval Package list: %s", name) + return + } + + if pack.NewVersion == "" { + // compare version: installed vs oval + vera := rpmver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) + verb := rpmver.NewVersion(ovalPackVer) + notFixedYet := false + if vera.LessThan(verb) { + notFixedYet = true + } + ps = append(ps, models.PackageStatus{ + Name: name, + NotFixedYet: notFixedYet, + }) + } else { + // compare version: newVer vs oval + packNewVer := fmt.Sprintf("%s-%s", pack.NewVersion, pack.NewRelease) + vera := rpmver.NewVersion(packNewVer) + verb := rpmver.NewVersion(ovalPackVer) + notFixedYet := false + if vera.LessThan(verb) { + notFixedYet = true + } + ps = append(ps, models.PackageStatus{ + Name: name, + NotFixedYet: notFixedYet, + }) + } + } + + default: + for k := range e.actuallyAffectedPackNames { + ps = append(ps, models.PackageStatus{ + Name: k, + }) + } + } + + return +} + +func (e *ovalResult) upsert(def ovalmodels.Definition, packName string) (upserted bool) { + for i, entry := range e.entries { + if entry.def.DefinitionID == def.DefinitionID { + e.entries[i].actuallyAffectedPackNames[packName] = true + return true + } + } + e.entries = append(e.entries, defPacks{ + def: def, + actuallyAffectedPackNames: map[string]bool{packName: true}, + }) + return false +} + +type request struct { + pack models.Package +} + +type response struct { + pack *models.Package + defs []ovalmodels.Definition +} + +// getDefsByPackNameViaHTTP fetches OVAL information via HTTP +func getDefsByPackNameViaHTTP(r *models.ScanResult) ( + relatedDefs ovalResult, err error) { + + reqChan := make(chan request, len(r.Packages)) + resChan := make(chan response, len(r.Packages)) + errChan := make(chan error, len(r.Packages)) + defer close(reqChan) + defer close(resChan) + defer close(errChan) + + go func() { + for _, pack := range r.Packages { + reqChan <- request{ + pack: pack, + } + } + }() + + concurrency := 10 + tasks := util.GenWorkers(concurrency) + for range r.Packages { + tasks <- func() { + select { + case req := <-reqChan: + url, err := util.URLPathJoin( + config.Conf.OvalDBURL, + "packs", + r.Family, + r.Release, + req.pack.Name, + ) + if err != nil { + errChan <- err + } else { + util.Log.Debugf("HTTP Request to %s", url) + httpGet(url, &req.pack, resChan, errChan) + } + } + } + } + + timeout := time.After(2 * 60 * time.Second) + var errs []error + for range r.Packages { + select { + case res := <-resChan: + for _, def := range res.defs { + for _, p := range def.AffectedPacks { + if res.pack.Name != p.Name { + continue + } + + if p.NotFixedYet { + relatedDefs.upsert(def, p.Name) + continue + } + + if less, err := lessThan(r.Family, *res.pack, p); err != nil { + util.Log.Debugf("Failed to parse versions: %s", err) + util.Log.Debugf("%#v\n%#v", *res.pack, p) + } else if less { + relatedDefs.upsert(def, p.Name) + } + } + } + case err := <-errChan: + errs = append(errs, err) + case <-timeout: + return relatedDefs, fmt.Errorf("Timeout Fetching OVAL") + } + } + if len(errs) != 0 { + return relatedDefs, fmt.Errorf("Failed to fetch OVAL. err: %v", errs) + } + return +} + +func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) { + var body string + var errs []error + var resp *http.Response + count, retryMax := 0, 3 + f := func() (err error) { + // resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() + resp, body, errs = gorequest.New().Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + count++ + if count == retryMax { + return nil + } + return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", + errs, url, resp) + } + return nil + } + notify := func(err error, t time.Duration) { + util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) + } + err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) + if err != nil { + errChan <- fmt.Errorf("HTTP Error %s", err) + return + } + if count == retryMax { + errChan <- fmt.Errorf("HRetry count exceeded") + return + } + + defs := []ovalmodels.Definition{} + if err := json.Unmarshal([]byte(body), &defs); err != nil { + errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", + body, err) + return + } + resChan <- response{ + pack: pack, + defs: defs, + } +} + +func getDefsByPackNameFromOvalDB(family, osRelease string, + installedPacks models.Packages) (relatedDefs ovalResult, err error) { + + ovallog.Initialize(config.Conf.LogDir) + path := config.Conf.OvalDBURL + if config.Conf.OvalDBType == "sqlite3" { + path = config.Conf.OvalDBPath + } + util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path) + + var ovaldb db.DB + if ovaldb, err = db.NewDB( + family, + config.Conf.OvalDBType, + path, + config.Conf.DebugSQL, + ); err != nil { + return + } + defer ovaldb.CloseDB() + for _, installedPack := range installedPacks { + definitions, err := ovaldb.GetByPackName(osRelease, installedPack.Name) + if err != nil { + return relatedDefs, fmt.Errorf("Failed to get %s OVAL info by package name: %v", family, err) + } + for _, def := range definitions { + for _, ovalPack := range def.AffectedPacks { + if installedPack.Name != ovalPack.Name { + continue + } + + if ovalPack.NotFixedYet { + relatedDefs.upsert(def, installedPack.Name) + continue + } + + less, err := lessThan(family, installedPack, ovalPack) + if err != nil { + util.Log.Debugf("Failed to parse versions: %s", err) + util.Log.Debugf("%#v\n%#v", installedPack, ovalPack) + } else if less { + relatedDefs.upsert(def, installedPack.Name) + } + } + } + } + return +} + +func lessThan(family string, packA models.Package, packB ovalmodels.Package) (bool, error) { + switch family { + case config.Debian, config.Ubuntu: + vera, err := debver.NewVersion(packA.Version) + if err != nil { + return false, err + } + verb, err := debver.NewVersion(packB.Version) + if err != nil { + return false, err + } + return vera.LessThan(verb), nil + case config.RedHat, config.CentOS, config.Oracle: + vera := rpmver.NewVersion(fmt.Sprintf("%s-%s", packA.Version, packA.Release)) + verb := rpmver.NewVersion(packB.Version) + return vera.LessThan(verb), nil + } + return false, fmt.Errorf("Package version comparison not supported: %s", family) +} diff --git a/oval/util_test.go b/oval/util_test.go new file mode 100644 index 00000000..241b16af --- /dev/null +++ b/oval/util_test.go @@ -0,0 +1,246 @@ +package oval + +import ( + "reflect" + "sort" + "testing" + + "github.com/future-architect/vuls/models" + ovalmodels "github.com/kotakanbe/goval-dictionary/models" +) + +func TestUpsert(t *testing.T) { + var tests = []struct { + res ovalResult + def ovalmodels.Definition + packName string + upserted bool + out ovalResult + }{ + //insert + { + res: ovalResult{}, + def: ovalmodels.Definition{ + DefinitionID: "1111", + }, + packName: "pack1", + upserted: false, + out: ovalResult{ + []defPacks{ + { + def: ovalmodels.Definition{ + DefinitionID: "1111", + }, + actuallyAffectedPackNames: map[string]bool{ + "pack1": true, + }, + }, + }, + }, + }, + //update + { + res: ovalResult{ + []defPacks{ + { + def: ovalmodels.Definition{ + DefinitionID: "1111", + }, + actuallyAffectedPackNames: map[string]bool{ + "pack1": true, + }, + }, + { + def: ovalmodels.Definition{ + DefinitionID: "2222", + }, + actuallyAffectedPackNames: map[string]bool{ + "pack3": true, + }, + }, + }, + }, + def: ovalmodels.Definition{ + DefinitionID: "1111", + }, + packName: "pack2", + upserted: true, + out: ovalResult{ + []defPacks{ + { + def: ovalmodels.Definition{ + DefinitionID: "1111", + }, + actuallyAffectedPackNames: map[string]bool{ + "pack1": true, + "pack2": true, + }, + }, + { + def: ovalmodels.Definition{ + DefinitionID: "2222", + }, + actuallyAffectedPackNames: map[string]bool{ + "pack3": true, + }, + }, + }, + }, + }, + } + for i, tt := range tests { + upserted := tt.res.upsert(tt.def, tt.packName) + if tt.upserted != upserted { + t.Errorf("[%d]\nexpected: %t\n actual: %t\n", i, tt.upserted, upserted) + } + if !reflect.DeepEqual(tt.out, tt.res) { + t.Errorf("[%d]\nexpected: %v\n actual: %v\n", i, tt.out, tt.res) + } + } +} + +func TestDefpacksToPackStatuses(t *testing.T) { + type in struct { + dp defPacks + family string + packs models.Packages + } + var tests = []struct { + in in + out models.PackageStatuses + }{ + // Ubuntu + { + in: in{ + family: "ubuntu", + packs: models.Packages{}, + dp: defPacks{ + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + NotFixedYet: true, + }, + { + Name: "b", + NotFixedYet: false, + }, + }, + }, + actuallyAffectedPackNames: map[string]bool{ + "a": true, + "b": true, + }, + }, + }, + out: models.PackageStatuses{ + { + Name: "a", + NotFixedYet: true, + }, + { + Name: "b", + NotFixedYet: false, + }, + }, + }, + + // RedHat, Amazon, Debian + { + in: in{ + family: "redhat", + packs: models.Packages{}, + dp: defPacks{ + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + }, + { + Name: "b", + }, + }, + }, + actuallyAffectedPackNames: map[string]bool{ + "a": true, + "b": true, + }, + }, + }, + out: models.PackageStatuses{ + { + Name: "a", + NotFixedYet: false, + }, + { + Name: "b", + NotFixedYet: false, + }, + }, + }, + + // CentOS + { + in: in{ + family: "centos", + packs: models.Packages{ + "a": {Version: "1.0.0"}, + "b": { + Version: "1.0.0", + NewVersion: "2.0.0", + }, + "c": { + Version: "1.0.0", + NewVersion: "1.5.0", + }, + }, + dp: defPacks{ + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + Version: "1.0.1", + }, + { + Name: "b", + Version: "1.5.0", + }, + { + Name: "c", + Version: "2.0.0", + }, + }, + }, + actuallyAffectedPackNames: map[string]bool{ + "a": true, + "b": true, + "c": true, + }, + }, + }, + out: models.PackageStatuses{ + { + Name: "a", + NotFixedYet: true, + }, + { + Name: "b", + NotFixedYet: false, + }, + { + Name: "c", + NotFixedYet: true, + }, + }, + }, + } + for i, tt := range tests { + actual := tt.in.dp.toPackStatuses(tt.in.family, tt.in.packs) + sort.Slice(actual, func(i, j int) bool { + return actual[i].Name < actual[j].Name + }) + if !reflect.DeepEqual(actual, tt.out) { + t.Errorf("[%d]\nexpected: %v\n actual: %v\n", i, tt.out, actual) + } + } +} diff --git a/report/azureblob.go b/report/azureblob.go index e8ac3578..f065accd 100644 --- a/report/azureblob.go +++ b/report/azureblob.go @@ -24,7 +24,7 @@ import ( "fmt" "time" - "github.com/Azure/azure-storage-go" + storage "github.com/Azure/azure-sdk-for-go/storage" c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" @@ -139,13 +139,9 @@ func createBlockBlob(cli storage.BlobStorageClient, k string, b []byte) error { k = k + ".gz" } - if err := cli.CreateBlockBlobFromReader( - c.Conf.AzureContainer, - k, - uint64(len(b)), - bytes.NewReader(b), - map[string]string{}, - ); err != nil { + ref := cli.GetContainerReference(c.Conf.AzureContainer) + blob := ref.GetBlobReference(k) + if err := blob.CreateBlockBlobFromReader(bytes.NewReader(b), nil); err != nil { return fmt.Errorf("Failed to upload data to %s/%s, %s", c.Conf.AzureContainer, k, err) } diff --git a/cveapi/cve_client.go b/report/cve_client.go similarity index 70% rename from cveapi/cve_client.go rename to report/cve_client.go index 972f7ae9..62d0036b 100644 --- a/cveapi/cve_client.go +++ b/report/cve_client.go @@ -15,13 +15,12 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package cveapi +package report import ( "encoding/json" "fmt" "net/http" - "sort" "time" "github.com/cenkalti/backoff" @@ -32,6 +31,7 @@ import ( cveconfig "github.com/kotakanbe/go-cve-dictionary/config" cvedb "github.com/kotakanbe/go-cve-dictionary/db" cve "github.com/kotakanbe/go-cve-dictionary/models" + log "github.com/sirupsen/logrus" ) // CveClient is api client of CVE disctionary service. @@ -46,10 +46,10 @@ func (api *cvedictClient) initialize() { api.baseURL = config.Conf.CveDBURL } -func (api cvedictClient) CheckHealth() (ok bool, err error) { - if config.Conf.CveDBURL == "" || config.Conf.CveDBType == "mysql" || config.Conf.CveDBType == "postgres" { +func (api cvedictClient) CheckHealth() error { + if !api.isFetchViaHTTP() { util.Log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType) - return true, nil + return nil } api.initialize() @@ -59,9 +59,10 @@ func (api cvedictClient) CheckHealth() (ok bool, err error) { resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() // resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End() if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { - return false, fmt.Errorf("Failed to request to CVE server. url: %s, errs: %v", url, errs) + return fmt.Errorf("Failed to request to CVE server. url: %s, errs: %v", + url, errs) } - return true, nil + return nil } type response struct { @@ -69,9 +70,8 @@ type response struct { CveDetail cve.CveDetail } -func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDetails, err error) { - switch config.Conf.CveDBType { - case "sqlite3", "mysql", "postgres": +func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails []*cve.CveDetail, err error) { + if !api.isFetchViaHTTP() { return api.FetchCveDetailsFromCveDB(cveIDs) } @@ -112,28 +112,26 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet select { case res := <-resChan: if len(res.CveDetail.CveID) == 0 { - cveDetails = append(cveDetails, cve.CveDetail{ + cveDetails = append(cveDetails, &cve.CveDetail{ CveID: res.Key, }) } else { - cveDetails = append(cveDetails, res.CveDetail) + cveDetails = append(cveDetails, &res.CveDetail) } case err := <-errChan: errs = append(errs, err) case <-timeout: - return []cve.CveDetail{}, fmt.Errorf("Timeout Fetching CVE") + return []*cve.CveDetail{}, fmt.Errorf("Timeout Fetching CVE") } } if len(errs) != 0 { - return []cve.CveDetail{}, + return []*cve.CveDetail{}, fmt.Errorf("Failed to fetch CVE. err: %v", errs) } - - sort.Sort(cveDetails) return } -func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails cve.CveDetails, err error) { +func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails []*cve.CveDetail, err error) { util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) cveconfig.Conf.DBType = config.Conf.CveDBType if config.Conf.CveDBType == "sqlite3" { @@ -142,23 +140,33 @@ func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails c cveconfig.Conf.DBPath = config.Conf.CveDBURL } cveconfig.Conf.DebugSQL = config.Conf.DebugSQL - if err := cvedb.OpenDB(); err != nil { - return []cve.CveDetail{}, + + var driver cvedb.DB + if driver, err = cvedb.NewDB(cveconfig.Conf.DBType); err != nil { + log.Error(err) + return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err) + } + + util.Log.Debugf("Opening DB (%s).", driver.Name()) + if err := driver.OpenDB( + cveconfig.Conf.DBType, + cveconfig.Conf.DBPath, + cveconfig.Conf.DebugSQL, + ); err != nil { + return []*cve.CveDetail{}, fmt.Errorf("Failed to open DB. err: %s", err) } + for _, cveID := range cveIDs { - cveDetail := cvedb.Get(cveID) + cveDetail := driver.Get(cveID) if len(cveDetail.CveID) == 0 { - cveDetails = append(cveDetails, cve.CveDetail{ + cveDetails = append(cveDetails, &cve.CveDetail{ CveID: cveID, }) } else { cveDetails = append(cveDetails, cveDetail) } } - - // order by CVE ID desc - sort.Sort(cveDetails) return } @@ -170,20 +178,25 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh // resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() resp, body, errs = gorequest.New().Get(url).End() if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { - return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", errs, url, resp) + return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", + errs, url, resp) } return nil } notify := func(err error, t time.Duration) { - util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) + util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", + t, err) } err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) if err != nil { errChan <- fmt.Errorf("HTTP Error %s", err) + return } cveDetail := cve.CveDetail{} if err := json.Unmarshal([]byte(body), &cveDetail); err != nil { - errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err) + errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", + body, err) + return } resChan <- response{ key, @@ -196,29 +209,37 @@ type responseGetCveDetailByCpeName struct { CveDetails []cve.CveDetail } -func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]cve.CveDetail, error) { - switch config.Conf.CveDBType { - case "sqlite3", "mysql", "postgres": - return api.FetchCveDetailsByCpeNameFromDB(cpeName) +func (api cvedictClient) isFetchViaHTTP() bool { + // Default value of CveDBType is sqlite3 + if config.Conf.CveDBURL != "" && config.Conf.CveDBType == "sqlite3" { + return true } - - api.baseURL = config.Conf.CveDBURL - url, err := util.URLPathJoin(api.baseURL, "cpes") - if err != nil { - return []cve.CveDetail{}, err - } - - query := map[string]string{"name": cpeName} - util.Log.Debugf("HTTP Request to %s, query: %#v", url, query) - return api.httpPost(cpeName, url, query) + return false } -func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cve.CveDetail, error) { +func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]*cve.CveDetail, error) { + if api.isFetchViaHTTP() { + api.baseURL = config.Conf.CveDBURL + url, err := util.URLPathJoin(api.baseURL, "cpes") + if err != nil { + return []*cve.CveDetail{}, err + } + + query := map[string]string{"name": cpeName} + util.Log.Debugf("HTTP Request to %s, query: %#v", url, query) + return api.httpPost(cpeName, url, query) + } + + return api.FetchCveDetailsByCpeNameFromDB(cpeName) +} + +func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]*cve.CveDetail, error) { var body string var errs []error var resp *http.Response f := func() (err error) { - req := gorequest.New().SetDebug(config.Conf.Debug).Post(url) + // req := gorequest.New().SetDebug(config.Conf.Debug).Post(url) + req := gorequest.New().Post(url) for key := range query { req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json") } @@ -233,18 +254,18 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c } err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) if err != nil { - return []cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err) + return []*cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err) } - cveDetails := []cve.CveDetail{} + cveDetails := []*cve.CveDetail{} if err := json.Unmarshal([]byte(body), &cveDetails); err != nil { - return []cve.CveDetail{}, + return []*cve.CveDetail{}, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err) } return cveDetails, nil } -func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.CveDetail, error) { +func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) (cveDetails []*cve.CveDetail, err error) { util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) cveconfig.Conf.DBType = config.Conf.CveDBType if config.Conf.CveDBType == "sqlite3" { @@ -254,9 +275,20 @@ func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.C } cveconfig.Conf.DebugSQL = config.Conf.DebugSQL - if err := cvedb.OpenDB(); err != nil { - return []cve.CveDetail{}, + var driver cvedb.DB + if driver, err = cvedb.NewDB(cveconfig.Conf.DBType); err != nil { + log.Error(err) + return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err) + } + + util.Log.Debugf("Opening DB (%s).", driver.Name()) + if err = driver.OpenDB( + cveconfig.Conf.DBType, + cveconfig.Conf.DBPath, + cveconfig.Conf.DebugSQL, + ); err != nil { + return []*cve.CveDetail{}, fmt.Errorf("Failed to open DB. err: %s", err) } - return cvedb.GetByCpeName(cpeName), nil + return driver.GetByCpeName(cpeName), nil } diff --git a/report/email.go b/report/email.go index 48d7449c..d3c1717d 100644 --- a/report/email.go +++ b/report/email.go @@ -35,14 +35,18 @@ type EMailWriter struct{} func (w EMailWriter) Write(rs ...models.ScanResult) (err error) { conf := config.Conf var message string - var totalResult models.ScanResult sender := NewEMailSender() + m := map[string]int{} for _, r := range rs { if conf.FormatOneEMail { message += formatFullPlainText(r) + "\r\n\r\n" - totalResult.KnownCves = append(totalResult.KnownCves, r.KnownCves...) - totalResult.UnknownCves = append(totalResult.UnknownCves, r.UnknownCves...) + + mm := r.ScannedCves.CountGroupBySeverity() + keys := []string{"High", "Medium", "Low", "Unknown"} + for _, k := range keys { + m[k] += mm[k] + } } else { var subject string if len(r.Errors) != 0 { @@ -50,7 +54,9 @@ func (w EMailWriter) Write(rs ...models.ScanResult) (err error) { conf.EMail.SubjectPrefix, r.ServerInfo()) } else { subject = fmt.Sprintf("%s%s %s", - conf.EMail.SubjectPrefix, r.ServerInfo(), r.CveSummary()) + conf.EMail.SubjectPrefix, + r.ServerInfo(), + r.ScannedCves.FormatCveSummary()) } message = formatFullPlainText(r) if err := sender.Send(subject, message); err != nil { @@ -59,6 +65,15 @@ func (w EMailWriter) Write(rs ...models.ScanResult) (err error) { } } + summary := "" + if config.Conf.IgnoreUnscoredCves { + summary = fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d)", + m["High"]+m["Medium"]+m["Low"], m["High"], m["Medium"], m["Low"]) + } + summary = fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d ?:%d)", + m["High"]+m["Medium"]+m["Low"]+m["Unknown"], + m["High"], m["Medium"], m["Low"], m["Unknown"]) + if conf.FormatOneEMail { message = fmt.Sprintf( ` @@ -71,9 +86,7 @@ One Line Summary formatOneLineSummary(rs...), message) subject := fmt.Sprintf("%s %s", - conf.EMail.SubjectPrefix, - totalResult.CveSummary(), - ) + conf.EMail.SubjectPrefix, summary) return sender.Send(subject, message) } return nil diff --git a/report/localfile.go b/report/localfile.go index 79f192a1..13e9b98f 100644 --- a/report/localfile.go +++ b/report/localfile.go @@ -58,8 +58,14 @@ func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) { } var b []byte - if b, err = json.Marshal(r); err != nil { - return fmt.Errorf("Failed to Marshal to JSON: %s", err) + if c.Conf.Debug { + if b, err = json.MarshalIndent(r, "", " "); err != nil { + return fmt.Errorf("Failed to Marshal to JSON: %s", err) + } + } else { + if b, err = json.Marshal(r); err != nil { + return fmt.Errorf("Failed to Marshal to JSON: %s", err) + } } if err := writeFile(p, b, 0600); err != nil { return fmt.Errorf("Failed to write JSON. path: %s, err: %s", p, err) diff --git a/report/report.go b/report/report.go new file mode 100644 index 00000000..df4fca4d --- /dev/null +++ b/report/report.go @@ -0,0 +1,227 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package report + +import ( + "fmt" + "strings" + "time" + + c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/oval" + "github.com/future-architect/vuls/util" +) + +const ( + vulsOpenTag = "" + vulsCloseTag = "" +) + +// FillCveInfos fills CVE Detailed Information +func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, error) { + var filled []models.ScanResult + reportedAt := time.Now() + for _, r := range rs { + if c.Conf.RefreshCve || needToRefreshCve(r) { + if err := FillCveInfo(&r); err != nil { + return nil, err + } + r.Lang = c.Conf.Lang + r.ReportedAt = reportedAt + r.Config.Report = c.Conf + r.Config.Report.Servers = map[string]c.ServerInfo{ + r.ServerName: c.Conf.Servers[r.ServerName], + } + if err := overwriteJSONFile(dir, r); err != nil { + return nil, fmt.Errorf("Failed to write JSON: %s", err) + } + filled = append(filled, r) + } else { + util.Log.Debugf("No need to refresh") + filled = append(filled, r) + } + } + + if c.Conf.Diff { + previous, err := loadPrevious(filled) + if err != nil { + return nil, err + } + + diff, err := diff(filled, previous) + if err != nil { + return nil, err + } + filled = []models.ScanResult{} + for _, r := range diff { + if err := fillCveDetail(&r); err != nil { + return nil, err + } + filled = append(filled, r) + } + } + + filtered := []models.ScanResult{} + for _, r := range filled { + r = r.FilterByCvssOver(c.Conf.CvssScoreOver) + r = r.FilterIgnoreCves(c.Conf.Servers[r.ServerName].IgnoreCves) + filtered = append(filtered, r) + } + return filtered, nil +} + +// FillCveInfo fill scanResult with cve info. +func FillCveInfo(r *models.ScanResult) error { + util.Log.Debugf("need to refresh") + + util.Log.Infof("Fill CVE detailed information with OVAL") + if err := FillWithOval(r); err != nil { + return fmt.Errorf("Failed to fill OVAL information: %s", err) + } + + util.Log.Infof("Fill CVE detailed information with CVE-DB") + if err := fillWithCveDB(r); err != nil { + return fmt.Errorf("Failed to fill CVE information: %s", err) + } + + for cveID := range r.ScannedCves { + vinfo := r.ScannedCves[cveID] + r.ScannedCves[cveID] = *vinfo.NilToEmpty() + } + return nil +} + +// fillCveDetail fetches NVD, JVN from CVE Database, and then set to fields. +func fillCveDetail(r *models.ScanResult) error { + var cveIDs []string + for _, v := range r.ScannedCves { + cveIDs = append(cveIDs, v.CveID) + } + + ds, err := CveClient.FetchCveDetails(cveIDs) + if err != nil { + return err + } + for _, d := range ds { + nvd := models.ConvertNvdToModel(d.CveID, d.Nvd) + jvn := models.ConvertJvnToModel(d.CveID, d.Jvn) + for cveID, vinfo := range r.ScannedCves { + if vinfo.CveID == d.CveID { + if vinfo.CveContents == nil { + vinfo.CveContents = models.CveContents{} + } + for _, con := range []models.CveContent{*nvd, *jvn} { + if !con.Empty() { + vinfo.CveContents[con.Type] = con + } + } + r.ScannedCves[cveID] = vinfo + break + } + } + } + return nil +} + +func fillWithCveDB(r *models.ScanResult) error { + sInfo := c.Conf.Servers[r.ServerName] + if err := fillVulnByCpeNames(sInfo.CpeNames, r.ScannedCves); err != nil { + return err + } + if err := fillCveDetail(r); err != nil { + return err + } + return nil +} + +// FillWithOval fetches OVAL database, and then set to fields. +func FillWithOval(r *models.ScanResult) (err error) { + var ovalClient oval.Client + var ovalFamily string + + // TODO + switch r.Family { + case c.Debian: + ovalClient = oval.NewDebian() + ovalFamily = c.Debian + case c.Ubuntu: + ovalClient = oval.NewUbuntu() + ovalFamily = c.Ubuntu + case c.RedHat: + ovalClient = oval.NewRedhat() + ovalFamily = c.RedHat + case c.CentOS: + ovalClient = oval.NewCentOS() + //use RedHat's OVAL + ovalFamily = c.RedHat + case c.Oracle: + ovalClient = oval.NewOracle() + ovalFamily = c.Oracle + case c.Amazon, c.Raspbian, c.FreeBSD: + return nil + default: + return fmt.Errorf("OVAL for %s is not implemented yet", r.Family) + } + + ok, err := ovalClient.CheckIfOvalFetched(ovalFamily, r.Release) + if err != nil { + return err + } + if !ok { + major := strings.Split(r.Release, ".")[0] + util.Log.Warnf("OVAL entries of %s %s are not found. It's recommended to use OVAL to improve scanning accuracy. For details, see https://github.com/kotakanbe/goval-dictionary#usage , Then report with --ovaldb-path or --ovaldb-url flag", ovalFamily, major) + return nil + } + + _, err = ovalClient.CheckIfOvalFresh(ovalFamily, r.Release) + if err != nil { + return err + } + + if err := ovalClient.FillWithOval(r); err != nil { + return err + } + return nil +} + +func fillVulnByCpeNames(cpeNames []string, scannedVulns models.VulnInfos) error { + for _, name := range cpeNames { + details, err := CveClient.FetchCveDetailsByCpeName(name) + if err != nil { + return err + } + for _, detail := range details { + if val, ok := scannedVulns[detail.CveID]; ok { + names := val.CpeNames + names = util.AppendIfMissing(names, name) + val.CpeNames = names + val.Confidence = models.CpeNameMatch + scannedVulns[detail.CveID] = val + } else { + v := models.VulnInfo{ + CveID: detail.CveID, + CpeNames: []string{name}, + Confidence: models.CpeNameMatch, + } + scannedVulns[detail.CveID] = v + } + } + } + return nil +} diff --git a/report/report_test.go b/report/report_test.go new file mode 100644 index 00000000..80c499fb --- /dev/null +++ b/report/report_test.go @@ -0,0 +1 @@ +package report diff --git a/report/s3.go b/report/s3.go index 1137324b..9fbaa6a7 100644 --- a/report/s3.go +++ b/report/s3.go @@ -22,6 +22,7 @@ import ( "encoding/json" "encoding/xml" "fmt" + "path" "time" "github.com/aws/aws-sdk-go/aws" @@ -147,8 +148,8 @@ func putObject(svc *s3.S3, k string, b []byte) error { } if _, err := svc.PutObject(&s3.PutObjectInput{ - Bucket: &c.Conf.S3Bucket, - Key: &k, + Bucket: aws.String(c.Conf.S3Bucket), + Key: aws.String(path.Join(c.Conf.S3ResultsDir, k)), Body: bytes.NewReader(b), }); err != nil { return fmt.Errorf("Failed to upload data to %s/%s, %s", diff --git a/report/slack.go b/report/slack.go index 434bc06b..141efbfd 100644 --- a/report/slack.go +++ b/report/slack.go @@ -24,11 +24,11 @@ import ( "strings" "time" - log "github.com/Sirupsen/logrus" "github.com/cenkalti/backoff" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/parnurzeal/gorequest" + log "github.com/sirupsen/logrus" ) type field struct { @@ -45,6 +45,7 @@ type attachment struct { Color string `json:"color"` Fields []*field `json:"fields"` MrkdwnIn []string `json:"mrkdwn_in"` + Footer string `json:"footer"` } type message struct { Text string `json:"text"` @@ -69,7 +70,8 @@ func (w SlackWriter) Write(rs ...models.ScanResult) error { if 0 < len(r.Errors) { serverInfo := fmt.Sprintf("*%s*", r.ServerInfo()) notifyUsers := getNotifyUsers(config.Conf.Slack.NotifyUsers) - txt := fmt.Sprintf("%s\n%s\nError: %s", notifyUsers, serverInfo, r.Errors) + txt := fmt.Sprintf("%s\n%s\nError: %s", + notifyUsers, serverInfo, r.Errors) msg := message{ Text: txt, Username: conf.AuthUser, @@ -152,54 +154,73 @@ func send(msg message) error { func msgText(r models.ScanResult) string { notifyUsers := "" - if 0 < len(r.KnownCves) || 0 < len(r.UnknownCves) { + if 0 < len(r.ScannedCves) { notifyUsers = getNotifyUsers(config.Conf.Slack.NotifyUsers) } serverInfo := fmt.Sprintf("*%s*", r.ServerInfo()) - return fmt.Sprintf("%s\n%s\n>%s", notifyUsers, serverInfo, r.CveSummary()) + return fmt.Sprintf("%s\n%s\n>%s", + notifyUsers, + serverInfo, + r.ScannedCves.FormatCveSummary()) } -func toSlackAttachments(scanResult models.ScanResult) (attaches []*attachment) { - cves := scanResult.KnownCves - if !config.Conf.IgnoreUnscoredCves { - cves = append(cves, scanResult.UnknownCves...) +func toSlackAttachments(r models.ScanResult) (attaches []*attachment) { + var vinfos []models.VulnInfo + if config.Conf.IgnoreUnscoredCves { + vinfos = r.ScannedCves.FindScoredVulns().ToSortedSlice() + } else { + vinfos = r.ScannedCves.ToSortedSlice() } - for _, cveInfo := range cves { - cveID := cveInfo.CveDetail.CveID - - curentPackages := []string{} - for _, p := range cveInfo.Packages { - curentPackages = append(curentPackages, p.ToStringCurrentVersion()) + for _, vinfo := range vinfos { + curent := []string{} + for _, affected := range vinfo.AffectedPackages { + if p, ok := r.Packages[affected.Name]; ok { + curent = append(curent, + fmt.Sprintf("%s-%s", p.Name, p.FormatVer())) + } else { + curent = append(curent, affected.Name) + } } - for _, n := range cveInfo.CpeNames { - curentPackages = append(curentPackages, n) + for _, n := range vinfo.CpeNames { + curent = append(curent, n) } - newPackages := []string{} - for _, p := range cveInfo.Packages { - newPackages = append(newPackages, p.ToStringNewVersion()) + new := []string{} + for _, affected := range vinfo.AffectedPackages { + if p, ok := r.Packages[affected.Name]; ok { + if affected.NotFixedYet { + new = append(new, "Not Fixed Yet") + } else { + new = append(new, p.FormatNewVer()) + } + } else { + new = append(new, "?") + } + } + for range vinfo.CpeNames { + new = append(new, "?") } a := attachment{ - Title: cveID, - TitleLink: fmt.Sprintf("%s/%s", nvdBaseURL, cveID), - Text: attachmentText(cveInfo, scanResult.Family), + Title: vinfo.CveID, + TitleLink: "https://nvd.nist.gov/vuln/detail/" + vinfo.CveID, + Text: attachmentText(vinfo, r.Family), MrkdwnIn: []string{"text", "pretext"}, Fields: []*field{ { - // Title: "Current Package/CPE", + // Title: "Current Package/CPE", Title: "Installed", - Value: strings.Join(curentPackages, "\n"), + Value: strings.Join(curent, "\n"), Short: true, }, { Title: "Candidate", - Value: strings.Join(newPackages, "\n"), + Value: strings.Join(new, "\n"), Short: true, }, }, - Color: color(cveInfo.CveDetail.CvssScore(config.Conf.Lang)), + Color: color(vinfo.MaxCvssScore().Value.Score), } attaches = append(attaches, &a) } @@ -220,68 +241,72 @@ func color(cvssScore float64) string { } } -func attachmentText(cveInfo models.CveInfo, osFamily string) string { - linkText := links(cveInfo, osFamily) - switch { - case config.Conf.Lang == "ja" && - 0 < cveInfo.CveDetail.Jvn.CvssScore(): +func attachmentText(vinfo models.VulnInfo, osFamily string) string { + maxCvss := vinfo.MaxCvssScore() + vectors := []string{} + for _, cvss := range vinfo.Cvss2Scores() { + calcURL := "" + switch cvss.Value.Type { + case models.CVSS2: + calcURL = fmt.Sprintf( + "https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=%s", + vinfo.CveID) + case models.CVSS3: + calcURL = fmt.Sprintf( + "https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=%s", + vinfo.CveID) + } - jvn := cveInfo.CveDetail.Jvn - return fmt.Sprintf("*%4.1f (%s)* <%s|%s>\n%s\n%s\n*Confidence:* %v", - cveInfo.CveDetail.CvssScore(config.Conf.Lang), - jvn.CvssSeverity(), - fmt.Sprintf(cvssV2CalcBaseURL, cveInfo.CveDetail.CveID), - jvn.CvssVector(), - jvn.CveTitle(), - linkText, - cveInfo.VulnInfo.Confidence, - ) - case 0 < cveInfo.CveDetail.CvssScore("en"): - nvd := cveInfo.CveDetail.Nvd - return fmt.Sprintf("*%4.1f (%s)* <%s|%s>\n%s\n%s\n*Confidence:* %v", - cveInfo.CveDetail.CvssScore(config.Conf.Lang), - nvd.CvssSeverity(), - fmt.Sprintf(cvssV2CalcBaseURL, cveInfo.CveDetail.CveID), - nvd.CvssVector(), - nvd.CveSummary(), - linkText, - cveInfo.VulnInfo.Confidence, - ) - default: - nvd := cveInfo.CveDetail.Nvd - return fmt.Sprintf("?\n%s\n%s\n*Confidence:* %v", - nvd.CveSummary(), linkText, cveInfo.VulnInfo.Confidence) - } -} + if cont, ok := vinfo.CveContents[cvss.Type]; ok { + v := fmt.Sprintf("<%s|%s> (<%s|%s>)", + calcURL, + cvss.Value.Format(), + cont.SourceLink, + cvss.Type) + vectors = append(vectors, v) -func links(cveInfo models.CveInfo, osFamily string) string { - links := []string{} + } else { + if 0 < len(vinfo.DistroAdvisories) { + links := []string{} + for k, v := range vinfo.VendorLinks(osFamily) { + links = append(links, fmt.Sprintf("<%s|%s>", + v, k)) + } - cweID := cveInfo.CveDetail.CweID() - if 0 < len(cweID) { - links = append(links, fmt.Sprintf("<%s|%s>", - cweURL(cweID), cweID)) - if config.Conf.Lang == "ja" { - links = append(links, fmt.Sprintf("<%s|%s(JVN)>", - cweJvnURL(cweID), cweID)) + v := fmt.Sprintf("<%s|%s> (%s)", + calcURL, + cvss.Value.Format(), + strings.Join(links, ", ")) + vectors = append(vectors, v) + } } } - cveID := cveInfo.CveDetail.CveID - if config.Conf.Lang == "ja" && 0 < len(cveInfo.CveDetail.Jvn.Link()) { - jvn := fmt.Sprintf("<%s|JVN>", cveInfo.CveDetail.Jvn.Link()) - links = append(links, jvn) + severity := strings.ToUpper(maxCvss.Value.Severity) + if severity == "" { + severity = "?" } - dlinks := distroLinks(cveInfo, osFamily) - for _, link := range dlinks { - links = append(links, - fmt.Sprintf("<%s|%s>", link.url, link.title)) - } - links = append(links, fmt.Sprintf("<%s|MITRE>", - fmt.Sprintf("%s%s", mitreBaseURL, cveID))) - links = append(links, fmt.Sprintf("<%s|CVEDetails>", - fmt.Sprintf("%s/%s", cveDetailsBaseURL, cveID))) + return fmt.Sprintf("*%4.1f (%s)* %s\n%s\n```%s```", + maxCvss.Value.Score, + severity, + cweIDs(vinfo, osFamily), + strings.Join(vectors, "\n"), + vinfo.Summaries(config.Conf.Lang, osFamily)[0].Value, + ) +} + +func cweIDs(vinfo models.VulnInfo, osFamily string) string { + links := []string{} + for _, cwe := range vinfo.CveContents.CweIDs(osFamily) { + if config.Conf.Lang == "ja" { + links = append(links, fmt.Sprintf("<%s|%s>", + cweJvnURL(cwe.Value), cwe.Value)) + } else { + links = append(links, fmt.Sprintf("<%s|%s>", + cweURL(cwe.Value), cwe.Value)) + } + } return strings.Join(links, " / ") } diff --git a/report/tui.go b/report/tui.go index ec74cf8a..59e89273 100644 --- a/report/tui.go +++ b/report/tui.go @@ -21,36 +21,41 @@ import ( "bytes" "fmt" "os" + "sort" "strings" "text/template" "time" - log "github.com/Sirupsen/logrus" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" "github.com/google/subcommands" "github.com/gosuri/uitable" "github.com/jroimartin/gocui" - cve "github.com/kotakanbe/go-cve-dictionary/models" + log "github.com/sirupsen/logrus" ) -var scanHistory models.ScanHistory +var scanResults models.ScanResults var currentScanResult models.ScanResult -var currentCveInfo int +var vinfos []models.VulnInfo +var currentVinfo int var currentDetailLimitY int +var currentChangelogLimitY int // RunTui execute main logic -func RunTui(history models.ScanHistory) subcommands.ExitStatus { - scanHistory = history +func RunTui(results models.ScanResults) subcommands.ExitStatus { + scanResults = results - g, err := gocui.NewGui(gocui.OutputNormal) - if err != nil { + // g, err := gocui.NewGui(gocui.OutputNormal) + g := gocui.NewGui() + if err := g.Init(); err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } defer g.Close() - g.SetManagerFunc(layout) + g.SetLayout(layout) + // g.SetManagerFunc(layout) if err := keybindings(g); err != nil { log.Errorf("%s", err) return subcommands.ExitFailure @@ -175,19 +180,19 @@ func nextView(g *gocui.Gui, v *gocui.View) error { var err error if v == nil { - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") } switch v.Name() { case "side": - _, err = g.SetCurrentView("summary") + err = g.SetCurrentView("summary") case "summary": - _, err = g.SetCurrentView("detail") + err = g.SetCurrentView("detail") case "detail": - _, err = g.SetCurrentView("changelog") + err = g.SetCurrentView("changelog") case "changelog": - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") default: - _, err = g.SetCurrentView("summary") + err = g.SetCurrentView("summary") } return err } @@ -196,19 +201,19 @@ func previousView(g *gocui.Gui, v *gocui.View) error { var err error if v == nil { - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") } switch v.Name() { case "side": - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") case "summary": - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") case "detail": - _, err = g.SetCurrentView("summary") + err = g.SetCurrentView("summary") case "changelog": - _, err = g.SetCurrentView("detail") + err = g.SetCurrentView("detail") default: - _, err = g.SetCurrentView("side") + err = g.SetCurrentView("side") } return err } @@ -216,27 +221,27 @@ func previousView(g *gocui.Gui, v *gocui.View) error { func movable(v *gocui.View, nextY int) (ok bool, yLimit int) { switch v.Name() { case "side": - yLimit = len(scanHistory.ScanResults) - 1 + yLimit = len(scanResults) - 1 if yLimit < nextY { return false, yLimit } return true, yLimit case "summary": - yLimit = len(currentScanResult.AllCves()) - 1 + yLimit = len(currentScanResult.ScannedCves) - 1 if yLimit < nextY { return false, yLimit } return true, yLimit case "detail": - if currentDetailLimitY < nextY { - return false, currentDetailLimitY - } + // if currentDetailLimitY < nextY { + // return false, currentDetailLimitY + // } return true, currentDetailLimitY case "changelog": - if currentDetailLimitY < nextY { - return false, currentDetailLimitY - } - return true, currentDetailLimitY + // if currentChangelogLimitY < nextY { + // return false, currentChangelogLimitY + // } + return true, currentChangelogLimitY default: return true, 0 } @@ -280,7 +285,7 @@ func cursorDown(g *gocui.Gui, v *gocui.View) error { // ok, := movable(v, oy+cy+1) // _, maxY := v.Size() ok, _ := movable(v, oy+cy+1) - // log.Info(cy, oy, maxY, yLimit) + // log.Info(cy, oy) if !ok { return nil } @@ -291,6 +296,10 @@ func cursorDown(g *gocui.Gui, v *gocui.View) error { } onMovingCursorRedrawView(g, v) } + + cx, cy := v.Cursor() + ox, oy := v.Origin() + debug(g, fmt.Sprintf("%v, %v, %v, %v", cx, cy, ox, oy)) return nil } @@ -387,7 +396,7 @@ func cursorPageUp(g *gocui.Gui, v *gocui.View) error { func previousSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if _, err := g.SetCurrentView("summary"); err != nil { + if err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -395,7 +404,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if _, err := g.SetCurrentView("detail"); err != nil { + if err := g.SetCurrentView("detail"); err != nil { return err } } @@ -405,7 +414,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { func nextSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if _, err := g.SetCurrentView("summary"); err != nil { + if err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -413,7 +422,7 @@ func nextSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if _, err := g.SetCurrentView("detail"); err != nil { + if err := g.SetCurrentView("detail"); err != nil { return err } } @@ -439,9 +448,10 @@ func changeHost(g *gocui.Gui, v *gocui.View) error { } serverName := strings.TrimSpace(l) - for _, r := range scanHistory.ScanResults { + for _, r := range scanResults { if serverName == strings.TrimSpace(r.ServerInfoTui()) { currentScanResult = r + vinfos = r.ScannedCves.ToSortedSlice() break } } @@ -495,7 +505,7 @@ func getLine(g *gocui.Gui, v *gocui.View) error { return err } fmt.Fprintln(v, l) - if _, err := g.SetCurrentView("msg"); err != nil { + if err := g.SetCurrentView("msg"); err != nil { return err } } @@ -510,14 +520,15 @@ func showMsg(g *gocui.Gui, v *gocui.View) error { // maxX, maxY := v.Size() _, maxY := v.Size() - l := fmt.Sprintf("cy: %d, oy: %d, maxY: %d, yLimit: %d, curCve %d, ok: %v", cy, oy, maxY, yLimit, currentCveInfo, ok) + l := fmt.Sprintf("cy: %d, oy: %d, maxY: %d, yLimit: %d, curCve %d, ok: %v", + cy, oy, maxY, yLimit, currentVinfo, ok) // if v, err := g.SetView("msg", maxX/2-30, maxY/2, maxX/2+30, maxY/2+2); err != nil { if v, err := g.SetView("msg", 10, maxY/2, 10+50, maxY/2+2); err != nil { if err != gocui.ErrUnknownView { return err } fmt.Fprintln(v, l) - if _, err := g.SetCurrentView("msg"); err != nil { + if err := g.SetCurrentView("msg"); err != nil { return err } } @@ -528,7 +539,7 @@ func delMsg(g *gocui.Gui, v *gocui.View) error { if err := g.DeleteView("msg"); err != nil { return err } - if _, err := g.SetCurrentView("summary"); err != nil { + if err := g.SetCurrentView("summary"); err != nil { return err } return nil @@ -551,6 +562,20 @@ func layout(g *gocui.Gui) error { if err := setChangelogLayout(g); err != nil { return err } + + return nil +} + +func debug(g *gocui.Gui, str string) error { + if config.Conf.Debug { + maxX, maxY := g.Size() + if _, err := g.View("debug"); err != gocui.ErrUnknownView { + g.DeleteView("debug") + } + if v, err := g.SetView("debug", maxX/2-7, maxY/2, maxX/2+7, maxY/2+2); err != nil { + fmt.Fprintf(v, str) + } + } return nil } @@ -562,14 +587,15 @@ func setSideLayout(g *gocui.Gui) error { } v.Highlight = true - for _, result := range scanHistory.ScanResults { + for _, result := range scanResults { fmt.Fprintln(v, result.ServerInfoTui()) } - if len(scanHistory.ScanResults) == 0 { + if len(scanResults) == 0 { return fmt.Errorf("No scan results") } - currentScanResult = scanHistory.ScanResults[0] - if _, err := g.SetCurrentView("side"); err != nil { + currentScanResult = scanResults[0] + vinfos = scanResults[0].ScannedCves.ToSortedSlice() + if err := g.SetCurrentView("side"); err != nil { return err } } @@ -603,50 +629,28 @@ func summaryLines() string { } indexFormat := "" - if len(currentScanResult.AllCves()) < 10 { + if len(currentScanResult.ScannedCves) < 10 { indexFormat = "[%1d]" - } else if len(currentScanResult.AllCves()) < 100 { + } else if len(currentScanResult.ScannedCves) < 100 { indexFormat = "[%2d]" } else { indexFormat = "[%3d]" } - for i, d := range currentScanResult.AllCves() { + for i, vinfo := range vinfos { + summary := vinfo.Titles( + config.Conf.Lang, currentScanResult.Family)[0].Value + cvssScore := fmt.Sprintf("| %4.1f", + vinfo.MaxCvssScore().Value.Score) + var cols []string - // packs := []string{} - // for _, pack := range d.Packages { - // packs = append(packs, pack.Name) - // } - if config.Conf.Lang == "ja" && 0 < d.CveDetail.Jvn.CvssScore() { - summary := d.CveDetail.Jvn.CveTitle() - cols = []string{ - fmt.Sprintf(indexFormat, i+1), - d.CveDetail.CveID, - fmt.Sprintf("| %4.1f", - d.CveDetail.CvssScore(config.Conf.Lang)), - fmt.Sprintf("| %3d |", d.VulnInfo.Confidence.Score), - summary, - } - } else { - summary := d.CveDetail.Nvd.CveSummary() - - var cvssScore string - if d.CveDetail.CvssScore("en") <= 0 { - cvssScore = "| ?" - } else { - cvssScore = fmt.Sprintf("| %4.1f", - d.CveDetail.CvssScore(config.Conf.Lang)) - } - - cols = []string{ - fmt.Sprintf(indexFormat, i+1), - d.CveDetail.CveID, - cvssScore, - fmt.Sprintf("| %3d |", d.VulnInfo.Confidence.Score), - summary, - } + cols = []string{ + fmt.Sprintf(indexFormat, i+1), + vinfo.CveID, + cvssScore, + fmt.Sprintf("| %3d |", vinfo.Confidence.Score), + summary, } - icols := make([]interface{}, len(cols)) for j := range cols { icols[j] = cols[j] @@ -665,7 +669,7 @@ func setDetailLayout(g *gocui.Gui) error { } _, cy := summaryView.Cursor() _, oy := summaryView.Origin() - currentCveInfo = cy + oy + currentVinfo = cy + oy if v, err := g.SetView("detail", -1, int(float64(maxY)*0.2), int(float64(maxX)*0.5), maxY); err != nil { if err != gocui.ErrUnknownView { @@ -693,22 +697,27 @@ func setChangelogLayout(g *gocui.Gui) error { } _, cy := summaryView.Cursor() _, oy := summaryView.Origin() - currentCveInfo = cy + oy + currentVinfo = cy + oy if v, err := g.SetView("changelog", int(float64(maxX)*0.5), int(float64(maxY)*0.2), maxX, maxY); err != nil { if err != gocui.ErrUnknownView { return err } - if len(currentScanResult.Errors) != 0 || len(currentScanResult.AllCves()) == 0 { + if len(currentScanResult.Errors) != 0 || len(currentScanResult.ScannedCves) == 0 { return nil } lines := []string{} - cveInfo := currentScanResult.AllCves()[currentCveInfo] - for _, pack := range cveInfo.Packages { + vinfo := vinfos[currentVinfo] + for _, adv := range vinfo.DistroAdvisories { + lines = append(lines, adv.Format()) + } + + for _, affected := range vinfo.AffectedPackages { + pack := currentScanResult.Packages[affected.Name] for _, p := range currentScanResult.Packages { if pack.Name == p.Name { - lines = append(lines, formatOneChangelog(p), "\n") + lines = append(lines, p.FormatChangelog(), "\n") } } } @@ -717,21 +726,19 @@ func setChangelogLayout(g *gocui.Gui) error { v.Editable = false v.Wrap = true - currentDetailLimitY = len(strings.Split(text, "\n")) - 1 + currentChangelogLimitY = len(strings.Split(text, "\n")) - 1 } return nil } type dataForTmpl struct { CveID string - CvssScore string - CvssVector string - CvssSeverity string + Cvsses string Summary string Confidence models.Confidence - CweURL string - VulnSiteLinks []string - References []cve.Reference + Cwes []models.CveContentStr + Links []string + References []models.Reference Packages []string CpeNames []string PublishedDate time.Time @@ -739,82 +746,75 @@ type dataForTmpl struct { } func detailLines() (string, error) { - if len(currentScanResult.Errors) != 0 { + r := currentScanResult + if len(r.Errors) != 0 { return "", nil } - if len(currentScanResult.AllCves()) == 0 { + if len(r.ScannedCves) == 0 { return "No vulnerable packages", nil } - cveInfo := currentScanResult.AllCves()[currentCveInfo] - cveID := cveInfo.CveDetail.CveID - - tmpl, err := template.New("detail").Parse(detailTemplate()) + tmpl, err := template.New("detail").Parse(mdTemplate) if err != nil { return "", err } - var cvssSeverity, cvssVector, summary string - var refs []cve.Reference - switch { - case config.Conf.Lang == "ja" && - 0 < cveInfo.CveDetail.Jvn.CvssScore(): - jvn := cveInfo.CveDetail.Jvn - cvssSeverity = jvn.CvssSeverity() - cvssVector = jvn.CvssVector() - summary = fmt.Sprintf("%s\n%s", jvn.CveTitle(), jvn.CveSummary()) - refs = jvn.VulnSiteReferences() - default: - nvd := cveInfo.CveDetail.Nvd - cvssSeverity = nvd.CvssSeverity() - cvssVector = nvd.CvssVector() - summary = nvd.CveSummary() - refs = nvd.VulnSiteReferences() + vinfo := vinfos[currentVinfo] + + packsVer := []string{} + vinfo.AffectedPackages.Sort() + for _, affected := range vinfo.AffectedPackages { + // packages detected by OVAL may not be actually installed + if pack, ok := r.Packages[affected.Name]; ok { + packsVer = append(packsVer, pack.FormatVersionFromTo(affected.NotFixedYet)) + } + } + sort.Strings(vinfo.CpeNames) + for _, name := range vinfo.CpeNames { + packsVer = append(packsVer, name) } - cweURL := cweURL(cveInfo.CveDetail.CweID()) - - links := []string{ - fmt.Sprintf("[NVD]( %s )", fmt.Sprintf("%s/%s", nvdBaseURL, cveID)), - fmt.Sprintf("[MITRE]( %s )", fmt.Sprintf("%s%s", mitreBaseURL, cveID)), - fmt.Sprintf("[CveDetais]( %s )", fmt.Sprintf("%s/%s", cveDetailsBaseURL, cveID)), - fmt.Sprintf("[CVSSv2 Calc]( %s )", fmt.Sprintf(cvssV2CalcBaseURL, cveID)), - fmt.Sprintf("[CVSSv3 Calc]( %s )", fmt.Sprintf(cvssV3CalcBaseURL, cveID)), - } - dlinks := distroLinks(cveInfo, currentScanResult.Family) - for _, link := range dlinks { - links = append(links, fmt.Sprintf("[%s]( %s )", link.title, link.url)) + links := []string{vinfo.CveContents.SourceLinks( + config.Conf.Lang, r.Family, vinfo.CveID)[0].Value, + vinfo.Cvss2CalcURL(), + vinfo.Cvss3CalcURL()} + for _, url := range vinfo.VendorLinks(r.Family) { + links = append(links, url) } - var cvssScore string - if cveInfo.CveDetail.CvssScore(config.Conf.Lang) == -1 { - cvssScore = "?" - } else { - cvssScore = fmt.Sprintf("%4.1f", cveInfo.CveDetail.CvssScore(config.Conf.Lang)) + refs := []models.Reference{} + for _, rr := range vinfo.CveContents.References(r.Family) { + for _, ref := range rr.Value { + refs = append(refs, ref) + } } - packages := []string{} - for _, pack := range cveInfo.Packages { - packages = append(packages, - fmt.Sprintf( - "%s -> %s", - pack.ToStringCurrentVersion(), - pack.ToStringNewVersion())) + summary := vinfo.Summaries(r.Lang, r.Family)[0] + + table := uitable.New() + table.MaxColWidth = maxColWidth + table.Wrap = true + scores := append(vinfo.Cvss3Scores(), vinfo.Cvss2Scores()...) + var cols []interface{} + for _, score := range scores { + cols = []interface{}{ + score.Value.Severity, + score.Value.Format(), + score.Type, + } + table.AddRow(cols...) } data := dataForTmpl{ - CveID: cveID, - CvssScore: cvssScore, - CvssSeverity: cvssSeverity, - CvssVector: cvssVector, - Summary: summary, - Confidence: cveInfo.VulnInfo.Confidence, - CweURL: cweURL, - VulnSiteLinks: links, - References: refs, - Packages: packages, - CpeNames: cveInfo.CpeNames, + CveID: vinfo.CveID, + Cvsses: fmt.Sprintf("%s\n", table), + Summary: fmt.Sprintf("%s (%s)", summary.Value, summary.Type), + Confidence: vinfo.Confidence, + Cwes: vinfo.CveContents.CweIDs(r.Family), + Links: util.Distinct(links), + Packages: packsVer, + References: refs, } buf := bytes.NewBuffer(nil) // create empty buffer @@ -825,52 +825,49 @@ func detailLines() (string, error) { return string(buf.Bytes()), nil } -func detailTemplate() string { - return ` +const mdTemplate = ` {{.CveID}} ============== -CVSS Score +CVSS Scores -------------- - -{{.CvssScore}} ({{.CvssSeverity}}) {{.CvssVector}} +{{.Cvsses }} Summary -------------- - {{.Summary }} -Confidence --------------- - {{.Confidence }} +Links +-------------- +{{range $link := .Links -}} +* {{$link}} +{{end}} CWE -------------- - - {{.CweURL }} +{{range .Cwes -}} +* {{.Value}} ({{.Type}}) +{{end}} Package/CPE -------------- - {{range $pack := .Packages -}} * {{$pack}} {{end -}} {{range $name := .CpeNames -}} * {{$name}} {{end}} -Links --------------- -{{range $link := .VulnSiteLinks -}} -* {{$link}} -{{end}} +Confidence +-------------- + {{.Confidence }} + + References -------------- - {{range .References -}} * [{{.Source}}]( {{.Link}} ) {{end}} ` -} diff --git a/report/util.go b/report/util.go index adf85f27..78330578 100644 --- a/report/util.go +++ b/report/util.go @@ -18,12 +18,19 @@ along with this program. If not, see . package report import ( - "bytes" + "encoding/json" "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "sort" "strings" + "time" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" "github.com/gosuri/uitable" ) @@ -39,7 +46,6 @@ func formatScanSummary(rs ...models.ScanResult) string { cols = []interface{}{ r.FormatServerName(), fmt.Sprintf("%s%s", r.Family, r.Release), - fmt.Sprintf("%d CVEs", len(r.ScannedCves)), r.Packages.FormatUpdatablePacksSummary(), } } else { @@ -64,7 +70,7 @@ func formatOneLineSummary(rs ...models.ScanResult) string { if len(r.Errors) == 0 { cols = []interface{}{ r.FormatServerName(), - r.CveSummary(), + r.ScannedCves.FormatCveSummary(), r.Packages.FormatUpdatablePacksSummary(), } } else { @@ -80,97 +86,69 @@ func formatOneLineSummary(rs ...models.ScanResult) string { } func formatShortPlainText(r models.ScanResult) string { - stable := uitable.New() - stable.MaxColWidth = maxColWidth - stable.Wrap = true - - cves := r.KnownCves - if !config.Conf.IgnoreUnscoredCves { - cves = append(cves, r.UnknownCves...) - } - - var buf bytes.Buffer - for i := 0; i < len(r.ServerInfo()); i++ { - buf.WriteString("=") - } - header := fmt.Sprintf("%s\n%s\n%s\t%s\n\n", - r.ServerInfo(), - buf.String(), - r.CveSummary(), - r.Packages.FormatUpdatablePacksSummary(), - ) - + header := r.FormatTextReportHeadedr() if len(r.Errors) != 0 { return fmt.Sprintf( "%s\nError: Scan with --debug to view the details\n%s\n\n", header, r.Errors) } - if len(cves) == 0 { + vulns := r.ScannedCves + if config.Conf.IgnoreUnscoredCves { + vulns = vulns.FindScoredVulns() + } + + if len(vulns) == 0 { return fmt.Sprintf(` %s No CVE-IDs are found in updatable packages. %s -`, header, r.Packages.FormatUpdatablePacksSummary()) + `, header, r.Packages.FormatUpdatablePacksSummary()) } - for _, d := range cves { - var packsVer string - for _, p := range d.Packages { - packsVer += fmt.Sprintf( - "%s -> %s\n", p.ToStringCurrentVersion(), p.ToStringNewVersion()) - } - for _, n := range d.CpeNames { - packsVer += n + stable := uitable.New() + stable.MaxColWidth = maxColWidth + stable.Wrap = true + for _, vuln := range vulns.ToSortedSlice() { + summaries := vuln.Summaries(config.Conf.Lang, r.Family) + links := vuln.CveContents.SourceLinks( + config.Conf.Lang, r.Family, vuln.CveID) + + vlinks := []string{} + for name, url := range vuln.VendorLinks(r.Family) { + vlinks = append(vlinks, fmt.Sprintf("%s (%s)", url, name)) } - var scols []string - switch { - case config.Conf.Lang == "ja" && - 0 < d.CveDetail.Jvn.CvssScore(): - summary := fmt.Sprintf("%s\n%s\n%s\n%sConfidence: %v", - d.CveDetail.Jvn.CveTitle(), - d.CveDetail.Jvn.Link(), - distroLinks(d, r.Family)[0].url, - packsVer, - d.VulnInfo.Confidence, - ) - scols = []string{ - d.CveDetail.CveID, - fmt.Sprintf("%-4.1f (%s)", - d.CveDetail.CvssScore(config.Conf.Lang), - d.CveDetail.Jvn.CvssSeverity(), - ), - summary, - } - - case 0 < d.CveDetail.CvssScore("en"): - summary := fmt.Sprintf("%s\n%s/%s\n%s\n%sConfidence: %v", - d.CveDetail.Nvd.CveSummary(), - cveDetailsBaseURL, - d.CveDetail.CveID, - distroLinks(d, r.Family)[0].url, - packsVer, - d.VulnInfo.Confidence, - ) - scols = []string{ - d.CveDetail.CveID, - fmt.Sprintf("%-4.1f (%s)", - d.CveDetail.CvssScore(config.Conf.Lang), - d.CveDetail.Nvd.CvssSeverity(), - ), - summary, - } - default: - summary := fmt.Sprintf("%s\n%sConfidence: %v", - distroLinks(d, r.Family)[0].url, packsVer, d.VulnInfo.Confidence) - scols = []string{ - d.CveDetail.CveID, - "?", - summary, - } + cvsses := "" + for _, cvss := range vuln.Cvss2Scores() { + cvsses += fmt.Sprintf("%s (%s)\n", cvss.Value.Format(), cvss.Type) + } + cvsses += vuln.Cvss2CalcURL() + "\n" + for _, cvss := range vuln.Cvss3Scores() { + cvsses += fmt.Sprintf("%s (%s)\n", cvss.Value.Format(), cvss.Type) + } + if 0 < len(vuln.Cvss3Scores()) { + cvsses += vuln.Cvss3CalcURL() + "\n" } + maxCvss := vuln.FormatMaxCvssScore() + rightCol := fmt.Sprintf(`%s +%s +--- +%s +%s +%sConfidence: %v`, + maxCvss, + summaries[0].Value, + links[0].Value, + strings.Join(vlinks, "\n"), + cvsses, + // packsVer, + vuln.Confidence, + ) + + leftCol := fmt.Sprintf("%s", vuln.CveID) + scols := []string{leftCol, rightCol} cols := make([]interface{}, len(scols)) for i := range cols { cols[i] = scols[i] @@ -182,295 +160,79 @@ No CVE-IDs are found in updatable packages. } func formatFullPlainText(r models.ScanResult) string { - serverInfo := r.ServerInfo() - - var buf bytes.Buffer - for i := 0; i < len(serverInfo); i++ { - buf.WriteString("=") - } - header := fmt.Sprintf("%s\n%s\n%s\t%s\n", - r.ServerInfo(), - buf.String(), - r.CveSummary(), - r.Packages.FormatUpdatablePacksSummary(), - ) - + header := r.FormatTextReportHeadedr() if len(r.Errors) != 0 { return fmt.Sprintf( "%s\nError: Scan with --debug to view the details\n%s\n\n", header, r.Errors) } - if len(r.KnownCves) == 0 && len(r.UnknownCves) == 0 { + vulns := r.ScannedCves + if config.Conf.IgnoreUnscoredCves { + vulns = vulns.FindScoredVulns() + } + + if len(vulns) == 0 { return fmt.Sprintf(` %s No CVE-IDs are found in updatable packages. %s -`, header, r.Packages.FormatUpdatablePacksSummary()) + `, header, r.Packages.FormatUpdatablePacksSummary()) } - scoredReport, unscoredReport := []string{}, []string{} - scoredReport, unscoredReport = formatPlainTextDetails(r, r.Family) + table := uitable.New() + table.MaxColWidth = maxColWidth + table.Wrap = true + for _, vuln := range vulns.ToSortedSlice() { + table.AddRow(vuln.CveID) + table.AddRow("----------------") + table.AddRow("Max Score", vuln.FormatMaxCvssScore()) + for _, cvss := range vuln.Cvss2Scores() { + table.AddRow(cvss.Type, cvss.Value.Format()) + } + for _, cvss := range vuln.Cvss3Scores() { + table.AddRow(cvss.Type, cvss.Value.Format()) + } + if 0 < len(vuln.Cvss2Scores()) { + table.AddRow("CVSSv2 Calc", vuln.Cvss2CalcURL()) + } + if 0 < len(vuln.Cvss3Scores()) { + table.AddRow("CVSSv3 Calc", vuln.Cvss3CalcURL()) + } + table.AddRow("Summary", vuln.Summaries( + config.Conf.Lang, r.Family)[0].Value) - unscored := "" - if !config.Conf.IgnoreUnscoredCves { - unscored = strings.Join(unscoredReport, "\n\n") - } + links := vuln.CveContents.SourceLinks( + config.Conf.Lang, r.Family, vuln.CveID) + table.AddRow("Source", links[0].Value) - scored := strings.Join(scoredReport, "\n\n") - detail := fmt.Sprintf(` -%s + vlinks := vuln.VendorLinks(r.Family) + for name, url := range vlinks { + table.AddRow(name, url) + } -%s -`, - scored, - unscored, - ) - return fmt.Sprintf("%s\n%s\n%s", header, detail, formatChangelogs(r)) -} + for _, v := range vuln.CveContents.CweIDs(r.Family) { + table.AddRow(fmt.Sprintf("%s (%s)", v.Value, v.Type), cweURL(v.Value)) + } -func formatPlainTextDetails(r models.ScanResult, osFamily string) (scoredReport, unscoredReport []string) { - for _, cve := range r.KnownCves { - switch config.Conf.Lang { - case "en": - if 0 < cve.CveDetail.Nvd.CvssScore() { - scoredReport = append( - scoredReport, formatPlainTextDetailsLangEn(cve, osFamily)) - } else { - scoredReport = append( - scoredReport, formatPlainTextUnknownCve(cve, osFamily)) - } - case "ja": - if 0 < cve.CveDetail.Jvn.CvssScore() { - scoredReport = append( - scoredReport, formatPlainTextDetailsLangJa(cve, osFamily)) - } else if 0 < cve.CveDetail.Nvd.CvssScore() { - scoredReport = append( - scoredReport, formatPlainTextDetailsLangEn(cve, osFamily)) - } else { - scoredReport = append( - scoredReport, formatPlainTextUnknownCve(cve, osFamily)) + packsVer := []string{} + vuln.AffectedPackages.Sort() + for _, affected := range vuln.AffectedPackages { + if pack, ok := r.Packages[affected.Name]; ok { + packsVer = append(packsVer, pack.FormatVersionFromTo(affected.NotFixedYet)) } } - } - for _, cve := range r.UnknownCves { - unscoredReport = append( - unscoredReport, formatPlainTextUnknownCve(cve, osFamily)) - } - return -} + sort.Strings(vuln.CpeNames) + for _, name := range vuln.CpeNames { + packsVer = append(packsVer, name) + } + table.AddRow("Package/CPE", strings.Join(packsVer, "\n")) + table.AddRow("Confidence", vuln.Confidence) -func formatPlainTextUnknownCve(cveInfo models.CveInfo, osFamily string) string { - cveID := cveInfo.CveDetail.CveID - dtable := uitable.New() - dtable.MaxColWidth = maxColWidth - dtable.Wrap = true - dtable.AddRow(cveID) - dtable.AddRow("-------------") - dtable.AddRow("Score", "?") - dtable.AddRow("NVD", fmt.Sprintf("%s/%s", nvdBaseURL, cveID)) - dlinks := distroLinks(cveInfo, osFamily) - for _, link := range dlinks { - dtable.AddRow(link.title, link.url) - } - dtable.AddRow("CVE Details", fmt.Sprintf("%s/%s", cveDetailsBaseURL, cveID)) - dtable = addPackageInfos(dtable, cveInfo.Packages) - dtable = addCpeNames(dtable, cveInfo.CpeNames) - dtable.AddRow("Confidence", cveInfo.VulnInfo.Confidence) - - return fmt.Sprintf("%s", dtable) -} - -func formatPlainTextDetailsLangJa(cveInfo models.CveInfo, osFamily string) string { - cveDetail := cveInfo.CveDetail - cveID := cveDetail.CveID - jvn := cveDetail.Jvn - - dtable := uitable.New() - dtable.MaxColWidth = maxColWidth - dtable.Wrap = true - dtable.AddRow(cveID) - dtable.AddRow("-------------") - if score := cveDetail.Jvn.CvssScore(); 0 < score { - dtable.AddRow("Score", - fmt.Sprintf("%4.1f (%s)", - cveDetail.Jvn.CvssScore(), - jvn.CvssSeverity(), - )) - } else { - dtable.AddRow("Score", "?") - } - dtable.AddRow("Vector", jvn.CvssVector()) - dtable.AddRow("Title", jvn.CveTitle()) - dtable.AddRow("Description", jvn.CveSummary()) - dtable.AddRow(cveDetail.CweID(), cweURL(cveDetail.CweID())) - dtable.AddRow(cveDetail.CweID()+"(JVN)", cweJvnURL(cveDetail.CweID())) - - dtable.AddRow("JVN", jvn.Link()) - dtable.AddRow("NVD", fmt.Sprintf("%s/%s", nvdBaseURL, cveID)) - dtable.AddRow("MITRE", fmt.Sprintf("%s%s", mitreBaseURL, cveID)) - dtable.AddRow("CVE Details", fmt.Sprintf("%s/%s", cveDetailsBaseURL, cveID)) - dtable.AddRow("CVSSv2 Clac", fmt.Sprintf(cvssV2CalcBaseURL, cveID)) - dtable.AddRow("CVSSv3 Clac", fmt.Sprintf(cvssV3CalcBaseURL, cveID)) - - dlinks := distroLinks(cveInfo, osFamily) - for _, link := range dlinks { - dtable.AddRow(link.title, link.url) + table.AddRow("\n") } - dtable = addPackageInfos(dtable, cveInfo.Packages) - dtable = addCpeNames(dtable, cveInfo.CpeNames) - dtable.AddRow("Confidence", cveInfo.VulnInfo.Confidence) - - return fmt.Sprintf("%s", dtable) -} - -func formatPlainTextDetailsLangEn(d models.CveInfo, osFamily string) string { - cveDetail := d.CveDetail - cveID := cveDetail.CveID - nvd := cveDetail.Nvd - - dtable := uitable.New() - dtable.MaxColWidth = maxColWidth - dtable.Wrap = true - dtable.AddRow(cveID) - dtable.AddRow("-------------") - - if score := cveDetail.Nvd.CvssScore(); 0 < score { - dtable.AddRow("Score", - fmt.Sprintf("%4.1f (%s)", - cveDetail.Nvd.CvssScore(), - nvd.CvssSeverity(), - )) - } else { - dtable.AddRow("Score", "?") - } - - dtable.AddRow("Vector", nvd.CvssVector()) - dtable.AddRow("Summary", nvd.CveSummary()) - dtable.AddRow("CWE", cweURL(cveDetail.CweID())) - - dtable.AddRow("NVD", fmt.Sprintf("%s/%s", nvdBaseURL, cveID)) - dtable.AddRow("MITRE", fmt.Sprintf("%s%s", mitreBaseURL, cveID)) - dtable.AddRow("CVE Details", fmt.Sprintf("%s/%s", cveDetailsBaseURL, cveID)) - dtable.AddRow("CVSSv2 Clac", fmt.Sprintf(cvssV2CalcBaseURL, cveID)) - dtable.AddRow("CVSSv3 Clac", fmt.Sprintf(cvssV3CalcBaseURL, cveID)) - - links := distroLinks(d, osFamily) - for _, link := range links { - dtable.AddRow(link.title, link.url) - } - dtable = addPackageInfos(dtable, d.Packages) - dtable = addCpeNames(dtable, d.CpeNames) - dtable.AddRow("Confidence", d.VulnInfo.Confidence) - - return fmt.Sprintf("%s\n", dtable) -} - -type distroLink struct { - title string - url string -} - -// distroLinks add Vendor URL of the CVE to table -func distroLinks(cveInfo models.CveInfo, osFamily string) []distroLink { - cveID := cveInfo.CveDetail.CveID - switch osFamily { - case "rhel", "centos": - links := []distroLink{ - { - "RHEL-CVE", - fmt.Sprintf("%s/%s", redhatSecurityBaseURL, cveID), - }, - } - for _, advisory := range cveInfo.DistroAdvisories { - aidURL := strings.Replace(advisory.AdvisoryID, ":", "-", -1) - links = append(links, distroLink{ - // "RHEL-errata", - advisory.AdvisoryID, - fmt.Sprintf(redhatRHSABaseBaseURL, aidURL), - }) - } - return links - case "oraclelinux": - links := []distroLink{ - { - "Oracle-CVE", - fmt.Sprintf(oracleSecurityBaseURL, cveID), - }, - } - for _, advisory := range cveInfo.DistroAdvisories { - links = append(links, distroLink{ - // "Oracle-ELSA" - advisory.AdvisoryID, - fmt.Sprintf(oracleELSABaseBaseURL, advisory.AdvisoryID), - }) - } - return links - case "amazon": - links := []distroLink{ - { - "RHEL-CVE", - fmt.Sprintf("%s/%s", redhatSecurityBaseURL, cveID), - }, - } - for _, advisory := range cveInfo.DistroAdvisories { - links = append(links, distroLink{ - // "Amazon-ALAS", - advisory.AdvisoryID, - fmt.Sprintf(amazonSecurityBaseURL, advisory.AdvisoryID), - }) - } - return links - case "ubuntu": - return []distroLink{ - { - "Ubuntu-CVE", - fmt.Sprintf("%s/%s", ubuntuSecurityBaseURL, cveID), - }, - //TODO Ubuntu USN - } - case "debian": - return []distroLink{ - { - "Debian-CVE", - fmt.Sprintf("%s/%s", debianTrackerBaseURL, cveID), - }, - // TODO Debian dsa - } - case "FreeBSD": - links := []distroLink{} - for _, advisory := range cveInfo.DistroAdvisories { - links = append(links, distroLink{ - "FreeBSD-VuXML", - fmt.Sprintf(freeBSDVuXMLBaseURL, advisory.AdvisoryID), - }) - } - return links - default: - return []distroLink{} - } -} - -// addPackageInfos add package information related the CVE to table -func addPackageInfos(table *uitable.Table, packs []models.PackageInfo) *uitable.Table { - for i, p := range packs { - var title string - if i == 0 { - title = "Package" - } - ver := fmt.Sprintf( - "%s -> %s", p.ToStringCurrentVersion(), p.ToStringNewVersion()) - table.AddRow(title, ver) - } - return table -} - -func addCpeNames(table *uitable.Table, names []string) *uitable.Table { - for _, n := range names { - table.AddRow("CPE", fmt.Sprintf("%s", n)) - } - return table + return fmt.Sprintf("%s\n%s", header, table) } func cweURL(cweID string) string { @@ -488,36 +250,263 @@ func formatChangelogs(r models.ScanResult) string { if p.NewVersion == "" { continue } - clog := formatOneChangelog(p) + clog := p.FormatChangelog() buf = append(buf, clog, "\n\n") } return strings.Join(buf, "\n") } -func formatOneChangelog(p models.PackageInfo) string { - buf := []string{} - if p.NewVersion == "" { - return "" +func needToRefreshCve(r models.ScanResult) bool { + if r.Lang != config.Conf.Lang { + return true } - packVer := fmt.Sprintf("%s -> %s", - p.ToStringCurrentVersion(), p.ToStringNewVersion()) - var delim bytes.Buffer - for i := 0; i < len(packVer); i++ { - delim.WriteString("-") + for _, cve := range r.ScannedCves { + if 0 < len(cve.CveContents) { + return false + } } - - clog := p.Changelog.Contents - if lines := strings.Split(clog, "\n"); len(lines) != 0 { - clog = strings.Join(lines[0:len(lines)-1], "\n") - } - - switch p.Changelog.Method { - case models.FailedToGetChangelog: - clog = "No changelogs" - case models.FailedToFindVersionInChangelog: - clog = "Failed to parse changelogs. For detials, check yourself" - } - buf = append(buf, packVer, delim.String(), clog) - return strings.Join(buf, "\n") + return true +} + +func overwriteJSONFile(dir string, r models.ScanResult) error { + before := config.Conf.FormatJSON + beforeDiff := config.Conf.Diff + config.Conf.FormatJSON = true + config.Conf.Diff = false + w := LocalFileWriter{CurrentDir: dir} + if err := w.Write(r); err != nil { + return fmt.Errorf("Failed to write summary report: %s", err) + } + config.Conf.FormatJSON = before + config.Conf.Diff = beforeDiff + return nil +} + +func loadPrevious(current models.ScanResults) (previous models.ScanResults, err error) { + dirs, err := ListValidJSONDirs() + if err != nil { + return + } + + for _, result := range current { + for _, dir := range dirs[1:] { + var r *models.ScanResult + path := filepath.Join(dir, result.ServerName+".json") + if r, err = loadOneServerScanResult(path); err != nil { + continue + } + if r.Family == result.Family && r.Release == result.Release { + previous = append(previous, *r) + util.Log.Infof("Privious json found: %s", path) + break + } + } + } + return previous, nil +} + +func diff(curResults, preResults models.ScanResults) (diffed models.ScanResults, err error) { + for _, current := range curResults { + found := false + var previous models.ScanResult + for _, r := range preResults { + if current.ServerName == r.ServerName { + found = true + previous = r + break + } + } + + if found { + current.ScannedCves = getDiffCves(previous, current) + packages := models.Packages{} + for _, s := range current.ScannedCves { + for _, affected := range s.AffectedPackages { + p := current.Packages[affected.Name] + packages[affected.Name] = p + } + } + current.Packages = packages + } + + diffed = append(diffed, current) + } + return diffed, err +} + +func getDiffCves(previous, current models.ScanResult) models.VulnInfos { + previousCveIDsSet := map[string]bool{} + for _, previousVulnInfo := range previous.ScannedCves { + previousCveIDsSet[previousVulnInfo.CveID] = true + } + + new := models.VulnInfos{} + updated := models.VulnInfos{} + for _, v := range current.ScannedCves { + if previousCveIDsSet[v.CveID] { + if isCveInfoUpdated(v.CveID, previous, current) { + updated[v.CveID] = v + } + } else { + new[v.CveID] = v + } + } + + for cveID, vuln := range new { + updated[cveID] = vuln + } + return updated +} + +func isCveInfoUpdated(cveID string, previous, current models.ScanResult) bool { + cTypes := []models.CveContentType{ + models.NVD, + models.JVN, + models.NewCveContentType(current.Family), + } + + prevLastModified := map[models.CveContentType]time.Time{} + for _, c := range previous.ScannedCves { + if cveID == c.CveID { + for _, cType := range cTypes { + content, _ := c.CveContents[cType] + prevLastModified[cType] = content.LastModified + } + break + } + } + + curLastModified := map[models.CveContentType]time.Time{} + for _, c := range current.ScannedCves { + if cveID == c.CveID { + for _, cType := range cTypes { + content, _ := c.CveContents[cType] + curLastModified[cType] = content.LastModified + } + break + } + } + for _, cType := range cTypes { + if equal := prevLastModified[cType].Equal(curLastModified[cType]); !equal { + return true + } + } + return false +} + +// jsonDirPattern is file name pattern of JSON directory +// 2016-11-16T10:43:28+09:00 +// 2016-11-16T10:43:28Z +var jsonDirPattern = regexp.MustCompile( + `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$`) + +// ListValidJSONDirs returns valid json directory as array +// Returned array is sorted so that recent directories are at the head +func ListValidJSONDirs() (dirs []string, err error) { + var dirInfo []os.FileInfo + if dirInfo, err = ioutil.ReadDir(config.Conf.ResultsDir); err != nil { + err = fmt.Errorf("Failed to read %s: %s", + config.Conf.ResultsDir, err) + return + } + for _, d := range dirInfo { + if d.IsDir() && jsonDirPattern.MatchString(d.Name()) { + jsonDir := filepath.Join(config.Conf.ResultsDir, d.Name()) + dirs = append(dirs, jsonDir) + } + } + sort.Slice(dirs, func(i, j int) bool { + return dirs[j] < dirs[i] + }) + return +} + +// JSONDir returns +// If there is an arg, check if it is a valid format and return the corresponding path under results. +// If arg passed via PIPE (such as history subcommand), return that path. +// Otherwise, returns the path of the latest directory +func JSONDir(args []string) (string, error) { + var err error + dirs := []string{} + + if 0 < len(args) { + if dirs, err = ListValidJSONDirs(); err != nil { + return "", err + } + + path := filepath.Join(config.Conf.ResultsDir, args[0]) + for _, d := range dirs { + ss := strings.Split(d, string(os.PathSeparator)) + timedir := ss[len(ss)-1] + if timedir == args[0] { + return path, nil + } + } + + return "", fmt.Errorf("Invalid path: %s", path) + } + + // PIPE + if config.Conf.Pipe { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return "", fmt.Errorf("Failed to read stdin: %s", err) + } + fields := strings.Fields(string(bytes)) + if 0 < len(fields) { + return filepath.Join(config.Conf.ResultsDir, fields[0]), nil + } + return "", fmt.Errorf("Stdin is invalid: %s", string(bytes)) + } + + // returns latest dir when no args or no PIPE + if dirs, err = ListValidJSONDirs(); err != nil { + return "", err + } + if len(dirs) == 0 { + return "", fmt.Errorf("No results under %s", + config.Conf.ResultsDir) + } + return dirs[0], nil +} + +// LoadScanResults read JSON data +func LoadScanResults(jsonDir string) (results models.ScanResults, err error) { + var files []os.FileInfo + if files, err = ioutil.ReadDir(jsonDir); err != nil { + return nil, fmt.Errorf("Failed to read %s: %s", jsonDir, err) + } + for _, f := range files { + if filepath.Ext(f.Name()) != ".json" || strings.HasSuffix(f.Name(), "_diff.json") { + continue + } + + var r *models.ScanResult + path := filepath.Join(jsonDir, f.Name()) + if r, err = loadOneServerScanResult(path); err != nil { + return nil, err + } + results = append(results, *r) + } + if len(results) == 0 { + return nil, fmt.Errorf("There is no json file under %s", jsonDir) + } + return +} + +// loadOneServerScanResult read JSON data of one server +func loadOneServerScanResult(jsonFile string) (*models.ScanResult, error) { + var ( + data []byte + err error + ) + if data, err = ioutil.ReadFile(jsonFile); err != nil { + return nil, fmt.Errorf("Failed to read %s: %s", jsonFile, err) + } + result := &models.ScanResult{} + if err := json.Unmarshal(data, result); err != nil { + return nil, fmt.Errorf("Failed to parse %s: %s", jsonFile, err) + } + return result, nil } diff --git a/report/util_test.go b/report/util_test.go new file mode 100644 index 00000000..4deb6934 --- /dev/null +++ b/report/util_test.go @@ -0,0 +1,327 @@ +package report + +import ( + "reflect" + "testing" + "time" + + "github.com/future-architect/vuls/models" + "github.com/k0kubun/pp" +) + +func TestIsCveInfoUpdated(t *testing.T) { + f := "2006-01-02" + old, _ := time.Parse(f, "2015-12-15") + new, _ := time.Parse(f, "2015-12-16") + + type In struct { + cveID string + cur models.ScanResult + prev models.ScanResult + } + var tests = []struct { + in In + expected bool + }{ + // NVD compare non-initialized times + { + in: In{ + cveID: "CVE-2017-0001", + cur: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0001", + LastModified: time.Time{}, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0001", + LastModified: time.Time{}, + }, + ), + }, + }, + }, + }, + expected: false, + }, + // JVN not updated + { + in: In{ + cveID: "CVE-2017-0002", + cur: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + }, + expected: false, + }, + // OVAL updated + { + in: In{ + cveID: "CVE-2017-0003", + cur: models.ScanResult{ + Family: "ubuntu", + ScannedCves: models.VulnInfos{ + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: new, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + Family: "ubuntu", + ScannedCves: models.VulnInfos{ + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + }, + expected: true, + }, + // OVAL newly detected + { + in: In{ + cveID: "CVE-2017-0004", + cur: models.ScanResult{ + Family: "redhat", + ScannedCves: models.VulnInfos{ + "CVE-2017-0004": { + CveID: "CVE-2017-0004", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + Family: "redhat", + ScannedCves: models.VulnInfos{}, + }, + }, + expected: true, + }, + } + for i, tt := range tests { + actual := isCveInfoUpdated(tt.in.cveID, tt.in.prev, tt.in.cur) + if actual != tt.expected { + t.Errorf("[%d] actual: %t, expected: %t", i, actual, tt.expected) + } + } +} + +func TestDiff(t *testing.T) { + atCurrent, _ := time.Parse("2006-01-02", "2014-12-31") + atPrevious, _ := time.Parse("2006-01-02", "2014-11-31") + var tests = []struct { + inCurrent models.ScanResults + inPrevious models.ScanResults + out models.ScanResult + }{ + { + inCurrent: models.ScanResults{ + { + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2012-6702": { + CveID: "CVE-2012-6702", + AffectedPackages: models.PackageStatuses{{Name: "libexpat1"}}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + "CVE-2014-9761": { + CveID: "CVE-2014-9761", + AffectedPackages: models.PackageStatuses{{Name: "libc-bin"}}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{}, + Errors: []string{}, + Optional: [][]interface{}{}, + }, + }, + inPrevious: models.ScanResults{ + { + ScannedAt: atPrevious, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2012-6702": { + CveID: "CVE-2012-6702", + AffectedPackages: models.PackageStatuses{{Name: "libexpat1"}}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + "CVE-2014-9761": { + CveID: "CVE-2014-9761", + AffectedPackages: models.PackageStatuses{{Name: "libc-bin"}}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{}, + Errors: []string{}, + Optional: [][]interface{}{}, + }, + }, + out: models.ScanResult{ + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + Packages: models.Packages{}, + ScannedCves: models.VulnInfos{}, + Errors: []string{}, + Optional: [][]interface{}{}, + }, + }, + { + inCurrent: models.ScanResults{ + { + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2016-6662": { + CveID: "CVE-2016-6662", + AffectedPackages: models.PackageStatuses{{Name: "mysql-libs"}}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{ + "mysql-libs": { + Name: "mysql-libs", + Version: "5.1.73", + Release: "7.el6", + NewVersion: "5.1.73", + NewRelease: "8.el6_8", + Repository: "", + Changelog: models.Changelog{ + Contents: "", + Method: "", + }, + }, + }, + }, + }, + inPrevious: models.ScanResults{ + { + ScannedAt: atPrevious, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{}, + }, + }, + out: models.ScanResult{ + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2016-6662": { + CveID: "CVE-2016-6662", + AffectedPackages: models.PackageStatuses{{Name: "mysql-libs"}}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{ + "mysql-libs": { + Name: "mysql-libs", + Version: "5.1.73", + Release: "7.el6", + NewVersion: "5.1.73", + NewRelease: "8.el6_8", + Repository: "", + Changelog: models.Changelog{ + Contents: "", + Method: "", + }, + }, + }, + }, + }, + } + + for i, tt := range tests { + diff, _ := diff(tt.inCurrent, tt.inPrevious) + for _, actual := range diff { + if !reflect.DeepEqual(actual.ScannedCves, tt.out.ScannedCves) { + h := pp.Sprint(actual.ScannedCves) + x := pp.Sprint(tt.out.ScannedCves) + t.Errorf("[%d] cves actual: \n %s \n expected: \n %s", i, h, x) + } + + for j := range tt.out.Packages { + if !reflect.DeepEqual(tt.out.Packages[j], actual.Packages[j]) { + h := pp.Sprint(tt.out.Packages[j]) + x := pp.Sprint(actual.Packages[j]) + t.Errorf("[%d] packages actual: \n %s \n expected: \n %s", i, x, h) + } + } + } + } +} diff --git a/report/writer.go b/report/writer.go index fa45a55b..66a760e0 100644 --- a/report/writer.go +++ b/report/writer.go @@ -24,28 +24,6 @@ import ( "github.com/future-architect/vuls/models" ) -const ( - nvdBaseURL = "https://nvd.nist.gov/vuln/detail" - mitreBaseURL = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" - cveDetailsBaseURL = "http://www.cvedetails.com/cve" - cvssV2CalcBaseURL = "https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=%s" - cvssV3CalcBaseURL = "https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=%s" - - redhatSecurityBaseURL = "https://access.redhat.com/security/cve" - redhatRHSABaseBaseURL = "https://rhn.redhat.com/errata/%s.html" - amazonSecurityBaseURL = "https://alas.aws.amazon.com/%s.html" - oracleSecurityBaseURL = "https://linux.oracle.com/cve/%s.html" - oracleELSABaseBaseURL = "https://linux.oracle.com/errata/%s.html" - - ubuntuSecurityBaseURL = "http://people.ubuntu.com/~ubuntu-security/cve" - debianTrackerBaseURL = "https://security-tracker.debian.org/tracker" - - freeBSDVuXMLBaseURL = "https://vuxml.freebsd.org/freebsd/%s.html" - - vulsOpenTag = "" - vulsCloseTag = "" -) - // ResultWriter Interface type ResultWriter interface { Write(...models.ScanResult) error diff --git a/scan/base.go b/scan/base.go index 99c4eb40..ba02a346 100644 --- a/scan/base.go +++ b/scan/base.go @@ -20,13 +20,12 @@ package scan import ( "fmt" "regexp" - "sort" "strings" "time" - "github.com/Sirupsen/logrus" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" + "github.com/sirupsen/logrus" ) type base struct { @@ -48,7 +47,7 @@ func (l *base) setServerInfo(c config.ServerInfo) { l.ServerInfo = c } -func (l base) getServerInfo() config.ServerInfo { +func (l *base) getServerInfo() config.ServerInfo { return l.ServerInfo } @@ -64,7 +63,7 @@ func (l *base) setDistro(fam, rel string) { l.setServerInfo(s) } -func (l base) getDistro() config.Distro { +func (l *base) getDistro() config.Distro { return l.Distro } @@ -72,11 +71,32 @@ func (l *base) setPlatform(p models.Platform) { l.Platform = p } -func (l base) getPlatform() models.Platform { +func (l *base) getPlatform() models.Platform { return l.Platform } -func (l base) allContainers() (containers []config.Container, err error) { +func (l *base) runningKernel() (release, version string, err error) { + r := l.exec("uname -r", noSudo) + if !r.isSuccess() { + return "", "", fmt.Errorf("Failed to SSH: %s", r) + } + release = strings.TrimSpace(r.Stdout) + + switch l.Distro.Family { + case config.Debian: + r := l.exec("uname -a", noSudo) + if !r.isSuccess() { + return "", "", fmt.Errorf("Failed to SSH: %s", r) + } + ss := strings.Fields(r.Stdout) + if 6 < len(ss) { + version = ss[6] + } + } + return +} + +func (l *base) allContainers() (containers []config.Container, err error) { switch l.ServerInfo.Containers.Type { case "", "docker": stdout, err := l.dockerPs("-a --format '{{.ID}} {{.Names}} {{.Image}}'") @@ -213,7 +233,7 @@ func (l *base) detectPlatform() { return } -func (l base) detectRunningOnAws() (ok bool, instanceID string, err error) { +func (l *base) detectRunningOnAws() (ok bool, instanceID string, err error) { if r := l.exec("type curl", noSudo); r.isSuccess() { cmd := "curl --max-time 1 --retry 3 --noproxy 169.254.169.254 http://169.254.169.254/latest/meta-data/instance-id" r := l.exec(cmd, noSudo) @@ -261,16 +281,11 @@ func (l base) detectRunningOnAws() (ok bool, instanceID string, err error) { // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/resource-ids.html var awsInstanceIDPattern = regexp.MustCompile(`^i-[0-9a-f]+$`) -func (l base) isAwsInstanceID(str string) bool { +func (l *base) isAwsInstanceID(str string) bool { return awsInstanceIDPattern.MatchString(str) } func (l *base) convertToModel() models.ScanResult { - for _, p := range l.VulnInfos { - sort.Sort(models.PackageInfosByName(p.Packages)) - } - sort.Sort(l.VulnInfos) - ctype := l.ServerInfo.Containers.Type if l.ServerInfo.Container.ContainerID != "" && ctype == "" { ctype = "docker" @@ -287,22 +302,19 @@ func (l *base) convertToModel() models.ScanResult { errs = append(errs, fmt.Sprintf("%s", e)) } - // Avoid null slice being null in JSON - for i := range l.VulnInfos { - l.VulnInfos[i].NilSliceToEmpty() - } - return models.ScanResult{ - ServerName: l.ServerInfo.ServerName, - ScannedAt: time.Now(), - Family: l.Distro.Family, - Release: l.Distro.Release, - Container: container, - Platform: l.Platform, - ScannedCves: l.VulnInfos, - Packages: l.Packages, - Optional: l.ServerInfo.Optional, - Errors: errs, + JSONVersion: models.JSONVersion, + ServerName: l.ServerInfo.ServerName, + ScannedAt: time.Now(), + Family: l.Distro.Family, + Release: l.Distro.Release, + Container: container, + Platform: l.Platform, + ScannedCves: l.VulnInfos, + RunningKernel: l.Kernel, + Packages: l.Packages, + Optional: l.ServerInfo.Optional, + Errors: errs, } } @@ -310,6 +322,6 @@ func (l *base) setErrs(errs []error) { l.errs = errs } -func (l base) getErrs() []error { +func (l *base) getErrs() []error { return l.errs } diff --git a/scan/debian.go b/scan/debian.go index 10bff021..581f0917 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -18,6 +18,7 @@ along with this program. If not, see . package scan import ( + "bufio" "fmt" "regexp" "strconv" @@ -28,6 +29,8 @@ import ( "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" + + "github.com/knqyf263/go-deb-version" ) // inherit OsTypeInterface @@ -37,7 +40,14 @@ type debian struct { // NewDebian is constructor func newDebian(c config.ServerInfo) *debian { - d := &debian{} + d := &debian{ + base: base{ + osPackages: osPackages{ + Packages: models.Packages{}, + VulnInfos: models.VulnInfos{}, + }, + }, + } d.log = util.NewCustomLogger(c) d.setServerInfo(c) return d @@ -53,8 +63,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err return false, deb, nil } if r.ExitStatus == 255 { - return false, deb, fmt.Errorf( - "Unable to connect via SSH. Check SSH settings. %s", r) + return false, deb, fmt.Errorf("Unable to connect via SSH. Check SSH settings. If you have never SSH to the host to be scanned, SSH to the host before scanning in order to add the HostKey. %s@%s port: %s\n%s", c.User, c.Host, c.Port, r) } util.Log.Debugf("Not Debian like Linux. %s", r) return false, deb, nil @@ -67,7 +76,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err // e.g. // Raspbian GNU/Linux 7 \n \l result := strings.Fields(r.Stdout) - if len(result) > 2 && result[0] == "Raspbian" { + if len(result) > 2 && result[0] == config.Raspbian { distro := strings.ToLower(trim(result[0])) deb.setDistro(distro, trim(result[2])) return true, deb, nil @@ -115,7 +124,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err // Debian cmd := "cat /etc/debian_version" if r := exec(c, cmd, noSudo); r.isSuccess() { - deb.setDistro("debian", trim(r.Stdout)) + deb.setDistro(config.Debian, trim(r.Stdout)) return true, deb, nil } @@ -128,60 +137,112 @@ func trim(str string) string { } func (o *debian) checkIfSudoNoPasswd() error { - cmd := util.PrependProxyEnv("apt-get update") - o.log.Infof("Checking... sudo %s", cmd) - r := o.exec(cmd, sudo) - if !r.isSuccess() { - o.log.Errorf("sudo error on %s", r) - return fmt.Errorf("Failed to sudo: %s", r) + if config.Conf.Deep || o.Distro.Family == config.Raspbian { + cmd := util.PrependProxyEnv("apt-get update") + o.log.Infof("Checking... sudo %s", cmd) + r := o.exec(cmd, sudo) + if !r.isSuccess() { + o.log.Errorf("sudo error on %s", r) + return fmt.Errorf("Failed to sudo: %s", r) + } + o.log.Infof("Sudo... Pass") + return nil } - o.log.Infof("Sudo... Pass") + + o.log.Infof("sudo ... No need") return nil } func (o *debian) checkDependencies() error { + packNames := []string{} + switch o.Distro.Family { - case "ubuntu", "raspbian": + case config.Ubuntu, config.Raspbian: + o.log.Infof("Dependencies... No need") return nil - case "debian": - // Debian needs aptitude to get changelogs. - // Because unable to get changelogs via apt-get changelog on Debian. - if r := o.exec("test -f /usr/bin/aptitude", noSudo); !r.isSuccess() { - msg := fmt.Sprintf("aptitude is not installed: %s", r) - o.log.Errorf(msg) - return fmt.Errorf(msg) + case config.Debian: + // https://askubuntu.com/a/742844 + packNames = append(packNames, "reboot-notifier") + + if !config.Conf.Deep { + // Debian needs aptitude to get changelogs. + // Because unable to get changelogs via apt-get changelog on Debian. + packNames = append(packNames, "aptitude") } - o.log.Infof("Dependencies... Pass") - return nil default: return fmt.Errorf("Not implemented yet: %s", o.Distro) } -} -func (o *debian) scanPackages() error { - var err error - var packs []models.PackageInfo - if packs, err = o.scanInstalledPackages(); err != nil { - o.log.Errorf("Failed to scan installed packages") - return err + for _, name := range packNames { + //TODO --show-format + cmd := "dpkg-query -W " + name + if r := o.exec(cmd, noSudo); !r.isSuccess() { + msg := fmt.Sprintf("%s is not installed", name) + o.log.Errorf(msg) + return fmt.Errorf(msg) + } } - o.setPackages(packs) - - var unsecurePacks []models.VulnInfo - if unsecurePacks, err = o.scanUnsecurePackages(packs); err != nil { - o.log.Errorf("Failed to scan vulnerable packages") - return err - } - o.setVulnInfos(unsecurePacks) + o.log.Infof("Dependencies... Pass") return nil } -func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error) { +func (o *debian) scanPackages() error { + // collect the running kernel information + release, version, err := o.runningKernel() + if err != nil { + o.log.Errorf("Failed to scan the running kernel version: %s", err) + return err + } + rebootRequired, err := o.rebootRequired() + if err != nil { + o.log.Errorf("Failed to detect the kernel reboot required: %s", err) + return err + } + o.Kernel = models.Kernel{ + Version: version, + Release: release, + RebootRequired: rebootRequired, + } + + installed, updatable, err := o.scanInstalledPackages() + if err != nil { + o.log.Errorf("Failed to scan installed packages: %s", err) + return err + } + o.Packages = installed + + if config.Conf.Deep || o.Distro.Family == config.Raspbian { + unsecures, err := o.scanUnsecurePackages(updatable) + if err != nil { + o.log.Errorf("Failed to scan vulnerable packages: %s", err) + return err + } + o.VulnInfos = unsecures + return nil + } + return nil +} + +// https://askubuntu.com/a/742844 +func (o *debian) rebootRequired() (bool, error) { + r := o.exec("test -f /var/run/reboot-required", noSudo) + switch r.ExitStatus { + case 0: + return true, nil + case 1: + return false, nil + default: + return false, fmt.Errorf("Failed to check reboot reauired: %s", r) + } +} + +func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, error) { + installed, updatable := models.Packages{}, models.Packages{} r := o.exec("dpkg-query -W", noSudo) if !r.isSuccess() { - return packs, fmt.Errorf("Failed to SSH: %s", r) + return nil, nil, fmt.Errorf("Failed to SSH: %s", r) } // e.g. @@ -192,16 +253,37 @@ func (o *debian) scanInstalledPackages() (packs []models.PackageInfo, err error) if trimmed := strings.TrimSpace(line); len(trimmed) != 0 { name, version, err := o.parseScannedPackagesLine(trimmed) if err != nil { - return nil, fmt.Errorf( + return nil, nil, fmt.Errorf( "Debian: Failed to parse package line: %s", line) } - packs = append(packs, models.PackageInfo{ + installed[name] = models.Package{ Name: name, Version: version, - }) + } } } - return + + updatableNames, err := o.getUpdatablePackNames() + if err != nil { + return nil, nil, err + } + for _, name := range updatableNames { + for _, pack := range installed { + if pack.Name == name { + updatable[name] = pack + break + } + } + } + + // Fill the candidate versions of upgradable packages + err = o.fillCandidateVersion(updatable) + if err != nil { + return nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err) + } + installed.MergeNewVersion(updatable) + + return installed, updatable, nil } var packageLinePattern = regexp.MustCompile(`^([^\t']+)\t(.+)$`) @@ -221,51 +303,33 @@ func (o *debian) parseScannedPackagesLine(line string) (name, version string, er return "", "", fmt.Errorf("Unknown format: %s", line) } -func (o *debian) scanUnsecurePackages(installed []models.PackageInfo) ([]models.VulnInfo, error) { +func (o *debian) aptGetUpdate() error { o.log.Infof("apt-get update...") cmd := util.PrependProxyEnv("apt-get update") if r := o.exec(cmd, sudo); !r.isSuccess() { - return nil, fmt.Errorf("Failed to SSH: %s", r) + return fmt.Errorf("Failed to SSH: %s", r) } + return nil +} - // Convert the name of upgradable packages to PackageInfo struct - upgradableNames, err := o.GetUpgradablePackNames() - if err != nil { - return nil, err - } - var upgradablePacks []models.PackageInfo - for _, name := range upgradableNames { - for _, pack := range installed { - if pack.Name == name { - upgradablePacks = append(upgradablePacks, pack) - break - } - } - } - - // Fill the candidate versions of upgradable packages - upgradablePacks, err = o.fillCandidateVersion(upgradablePacks) - if err != nil { - return nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err) - } - - o.Packages.MergeNewVersion(upgradablePacks) +func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) { + o.aptGetUpdate() // Setup changelog cache current := cache.Meta{ Name: o.getServerInfo().GetServerName(), Distro: o.getServerInfo().Distro, - Packs: upgradablePacks, + Packs: updatable, } o.log.Debugf("Ensure changelog cache: %s", current.Name) - var meta *cache.Meta - if meta, err = o.ensureChangelogCache(current); err != nil { + meta, err := o.ensureChangelogCache(current) + if err != nil { return nil, err } // Collect CVE information of upgradable packages - vulnInfos, err := o.scanVulnInfos(upgradablePacks, meta) + vulnInfos, err := o.scanVulnInfos(updatable, meta) if err != nil { return nil, fmt.Errorf("Failed to scan unsecure packages. err: %s", err) } @@ -308,34 +372,34 @@ func (o *debian) ensureChangelogCache(current cache.Meta) (*cache.Meta, error) { return &cached, nil } -func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []models.PackageInfo, err error) { +func (o *debian) fillCandidateVersion(updatables models.Packages) (err error) { names := []string{} - for _, p := range before { - names = append(names, p.Name) + for name := range updatables { + names = append(names, name) } cmd := fmt.Sprintf("LANGUAGE=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " ")) r := o.exec(cmd, noSudo) if !r.isSuccess() { - return nil, fmt.Errorf("Failed to SSH: %s", r) + return fmt.Errorf("Failed to SSH: %s", r) } packChangelog := o.splitAptCachePolicy(r.Stdout) for k, v := range packChangelog { ver, err := o.parseAptCachePolicy(v, k) if err != nil { - return nil, fmt.Errorf("Failed to parse %s", err) + return fmt.Errorf("Failed to parse %s", err) } - p, found := before.FindByName(k) - if !found { - return nil, fmt.Errorf("Not found: %s", k) + pack, ok := updatables[k] + if !ok { + return fmt.Errorf("Not found: %s", k) } - p.NewVersion = ver.Candidate - filled = append(filled, p) + pack.NewVersion = ver.Candidate + updatables[k] = pack } return } -func (o *debian) GetUpgradablePackNames() (packNames []string, err error) { - cmd := util.PrependProxyEnv("LANGUAGE=en_US.UTF-8 apt-get upgrade --dry-run") +func (o *debian) getUpdatablePackNames() (packNames []string, err error) { + cmd := util.PrependProxyEnv("LANGUAGE=en_US.UTF-8 apt-get dist-upgrade --dry-run") r := o.exec(cmd, noSudo) if r.isSuccess(0, 1) { return o.parseAptGetUpgrade(r.Stdout) @@ -345,7 +409,7 @@ func (o *debian) GetUpgradablePackNames() (packNames []string, err error) { cmd, r.ExitStatus, r.Stdout, r.Stderr) } -func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, err error) { +func (o *debian) parseAptGetUpgrade(stdout string) (updatableNames []string, err error) { startRe := regexp.MustCompile(`The following packages will be upgraded:`) stopRe := regexp.MustCompile(`^(\d+) upgraded.*`) startLineFound, stopLineFound := false, false @@ -360,21 +424,21 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er } result := stopRe.FindStringSubmatch(line) if len(result) == 2 { - numUpgradablePacks, err := strconv.Atoi(result[1]) + nUpdatable, err := strconv.Atoi(result[1]) if err != nil { return nil, fmt.Errorf( "Failed to scan upgradable packages number. line: %s", line) } - if numUpgradablePacks != len(upgradableNames) { + if nUpdatable != len(updatableNames) { return nil, fmt.Errorf( "Failed to scan upgradable packages, expected: %s, detected: %d", - result[1], len(upgradableNames)) + result[1], len(updatableNames)) } stopLineFound = true o.log.Debugf("Found the stop line. line: %s", line) break } - upgradableNames = append(upgradableNames, strings.Fields(line)...) + updatableNames = append(updatableNames, strings.Fields(line)...) } if !startLineFound { // no upgrades @@ -387,19 +451,28 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er return } -func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache.Meta) (models.VulnInfos, error) { - resChan := make(chan struct { - models.PackageInfo - DetectedCveIDs - }, len(upgradablePacks)) - errChan := make(chan error, len(upgradablePacks)) - reqChan := make(chan models.PackageInfo, len(upgradablePacks)) +// DetectedCveID has CveID, Confidence and DetectionMethod fields +// LenientMatching will be true if this vulnerability is not detected by accurate version matching. +// see https://github.com/future-architect/vuls/pull/328 +type DetectedCveID struct { + CveID string + Confidence models.Confidence +} + +func (o *debian) scanVulnInfos(updatablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) { + type response struct { + pack *models.Package + DetectedCveIDs []DetectedCveID + } + resChan := make(chan response, len(updatablePacks)) + errChan := make(chan error, len(updatablePacks)) + reqChan := make(chan models.Package, len(updatablePacks)) defer close(resChan) defer close(errChan) defer close(reqChan) go func() { - for _, pack := range upgradablePacks { + for _, pack := range updatablePacks { reqChan <- pack } }() @@ -407,50 +480,53 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache timeout := time.After(30 * 60 * time.Second) concurrency := 10 tasks := util.GenWorkers(concurrency) - for range upgradablePacks { + for range updatablePacks { tasks <- func() { select { case pack := <-reqChan: - func(p models.PackageInfo) { + func(p models.Package) { changelog := o.getChangelogCache(meta, p) if 0 < len(changelog) { - cveIDs, _ := o.getCveIDsFromChangelog(changelog, p.Name, p.Version) - resChan <- struct { - models.PackageInfo - DetectedCveIDs - }{p, cveIDs} + cveIDs, pack := o.getCveIDsFromChangelog(changelog, p.Name, p.Version) + resChan <- response{pack, cveIDs} return } // if the changelog is not in cache or failed to get from local cache, // get the changelog of the package via internet. // After that, store it in the cache. - if cveIDs, err := o.scanPackageCveIDs(p); err != nil { + if cveIDs, pack, err := o.scanPackageCveIDs(p); err != nil { errChan <- err } else { - resChan <- struct { - models.PackageInfo - DetectedCveIDs - }{p, cveIDs} + resChan <- response{pack, cveIDs} } }(pack) } } } - // { DetectedCveID{} : [packageInfo] } - cvePackages := make(map[DetectedCveID][]models.PackageInfo) + // { DetectedCveID{} : [package] } + cvePackages := make(map[DetectedCveID][]string) errs := []error{} - for i := 0; i < len(upgradablePacks); i++ { + for i := 0; i < len(updatablePacks); i++ { select { - case pair := <-resChan: - pack := pair.PackageInfo - cveIDs := pair.DetectedCveIDs - for _, cveID := range cveIDs { - cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack) + case response := <-resChan: + if response.pack == nil { + continue } - o.log.Infof("(%d/%d) Scanned %s-%s : %s", - i+1, len(upgradablePacks), pair.Name, pair.PackageInfo.Version, cveIDs) + o.Packages[response.pack.Name] = *response.pack + cves := response.DetectedCveIDs + for _, cve := range cves { + packNames, ok := cvePackages[cve] + if ok { + packNames = append(packNames, response.pack.Name) + } else { + packNames = []string{response.pack.Name} + } + cvePackages[cve] = packNames + } + o.log.Infof("(%d/%d) Scanned %s: %s", + i+1, len(updatablePacks), response.pack.Name, cves) case err := <-errChan: errs = append(errs, err) case <-timeout: @@ -466,17 +542,22 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache cveIDs = append(cveIDs, k) } o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs) - var vinfos models.VulnInfos - for k, v := range cvePackages { - vinfos = append(vinfos, models.VulnInfo{ - CveID: k.CveID, - Confidence: k.Confidence, - Packages: v, - }) + vinfos := models.VulnInfos{} + for cveID, names := range cvePackages { + affected := models.PackageStatuses{} + for _, n := range names { + affected = append(affected, models.PackageStatus{Name: n}) + } + + vinfos[cveID.CveID] = models.VulnInfo{ + CveID: cveID.CveID, + Confidence: cveID.Confidence, + AffectedPackages: affected, + } } // Update meta package information of changelog cache to the latest one. - meta.Packs = upgradablePacks + meta.Packs = updatablePacks if err := cache.DB.RefreshMeta(*meta); err != nil { return nil, err } @@ -484,10 +565,10 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.PackageInfo, meta *cache return vinfos, nil } -func (o *debian) getChangelogCache(meta *cache.Meta, pack models.PackageInfo) string { - cachedPack, found := meta.FindPack(pack.Name) +func (o *debian) getChangelogCache(meta *cache.Meta, pack models.Package) string { + cachedPack, found := meta.Packs[pack.Name] if !found { - o.log.Debugf("Not found: %s", pack.Name) + o.log.Debugf("Not found in cache: %s", pack.Name) return "" } @@ -512,12 +593,12 @@ func (o *debian) getChangelogCache(meta *cache.Meta, pack models.PackageInfo) st return changelog } -func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]DetectedCveID, error) { +func (o *debian) scanPackageCveIDs(pack models.Package) ([]DetectedCveID, *models.Package, error) { cmd := "" switch o.Distro.Family { - case "ubuntu", "raspbian": + case config.Ubuntu, config.Raspbian: cmd = fmt.Sprintf(`PAGER=cat apt-get -q=2 changelog %s`, pack.Name) - case "debian": + case config.Debian: cmd = fmt.Sprintf(`PAGER=cat aptitude -q=2 changelog %s`, pack.Name) } cmd = util.PrependProxyEnv(cmd) @@ -526,151 +607,146 @@ func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]DetectedCveID, er if !r.isSuccess() { o.log.Warnf("Failed to SSH: %s", r) // Ignore this Error. - return nil, nil + return nil, nil, nil } stdout := strings.Replace(r.Stdout, "\r", "", -1) - cveIDs, clog := o.getCveIDsFromChangelog( - stdout, pack.Name, pack.Version) + cveIDs, clogFilledPack := o.getCveIDsFromChangelog(stdout, pack.Name, pack.Version) - if clog.Method != models.FailedToGetChangelog { - err := cache.DB.PutChangelog(o.getServerInfo().GetServerName(), pack.Name, clog.Contents) + if clogFilledPack.Changelog.Method != models.FailedToGetChangelog { + err := cache.DB.PutChangelog( + o.getServerInfo().GetServerName(), pack.Name, pack.Changelog.Contents) if err != nil { - return nil, fmt.Errorf("Failed to put changelog into cache") + return nil, nil, fmt.Errorf("Failed to put changelog into cache") } } // No error will be returned. Only logging. - return cveIDs, nil + return cveIDs, clogFilledPack, nil } -// Debian Version Numbers -// https://readme.phys.ethz.ch/documentation/debian_version_numbers/ func (o *debian) getCveIDsFromChangelog( - changelog, name, ver string) ([]DetectedCveID, models.Changelog) { + changelog, name, ver string) ([]DetectedCveID, *models.Package) { - if cveIDs, relevant, err := o.parseChangelog( + if cveIDs, pack, err := o.parseChangelog( changelog, name, ver, models.ChangelogExactMatch); err == nil { - return cveIDs, relevant + return cveIDs, pack } var verAfterColon string - var err error splittedByColon := strings.Split(ver, ":") if 1 < len(splittedByColon) { verAfterColon = splittedByColon[1] - if cveIDs, relevant, err := o.parseChangelog( + if cveIDs, pack, err := o.parseChangelog( changelog, name, verAfterColon, models.ChangelogLenientMatch); err == nil { - return cveIDs, relevant + return cveIDs, pack } } delim := []string{"+", "~", "build"} switch o.Distro.Family { - case "ubuntu": - delim = append(delim, "ubuntu") - case "debian": - case "Raspbian": + case config.Ubuntu: + delim = append(delim, config.Ubuntu) + case config.Debian: + case config.Raspbian: } for _, d := range delim { ss := strings.Split(ver, d) if 1 < len(ss) { - if cveIDs, relevant, err := o.parseChangelog( + if cveIDs, pack, err := o.parseChangelog( changelog, name, ss[0], models.ChangelogLenientMatch); err == nil { - return cveIDs, relevant + return cveIDs, pack } } ss = strings.Split(verAfterColon, d) if 1 < len(ss) { - if cveIDs, relevant, err := o.parseChangelog( + if cveIDs, pack, err := o.parseChangelog( changelog, name, ss[0], models.ChangelogLenientMatch); err == nil { - return cveIDs, relevant + return cveIDs, pack } } } // Only logging the error. - o.log.Error(err) - - for i, p := range o.Packages { - if p.Name == name { - o.Packages[i].Changelog = models.Changelog{ - Contents: "", - Method: models.FailedToFindVersionInChangelog, - } - } - } + o.log.Warnf("Failed to find the version in changelog: %s-%s", name, ver) + o.log.Debugf("Changelog of : %s-%s", name, ver, changelog) // If the version is not in changelog, return entire changelog to put into cache - return []DetectedCveID{}, models.Changelog{ + pack := o.Packages[name] + pack.Changelog = models.Changelog{ Contents: changelog, Method: models.FailedToFindVersionInChangelog, } -} -// DetectedCveID has CveID, Confidence and DetectionMethod fields -// LenientMatching will be true if this vulnerability is not detected by accurate version matching. -// see https://github.com/future-architect/vuls/pull/328 -type DetectedCveID struct { - CveID string - Confidence models.Confidence + return []DetectedCveID{}, &pack } -// DetectedCveIDs is a slice of DetectedCveID -type DetectedCveIDs []DetectedCveID - var cveRe = regexp.MustCompile(`(CVE-\d{4}-\d{4,})`) // Collect CVE-IDs included in the changelog. -// The version which specified in argument(versionOrLater) is excluded. -func (o *debian) parseChangelog(changelog, name, ver string, confidence models.Confidence) ([]DetectedCveID, models.Changelog, error) { +// The version specified in argument(versionOrLater) is used to compare. +func (o *debian) parseChangelog(changelog, name, ver string, confidence models.Confidence) ([]DetectedCveID, *models.Package, error) { + installedVer, err := version.NewVersion(ver) + if err != nil { + return nil, nil, fmt.Errorf("Failed to parse installed version: %s, %s", ver, err) + } buf, cveIDs := []string{}, []string{} - stopRe := regexp.MustCompile(fmt.Sprintf(`\(%s\)`, regexp.QuoteMeta(ver))) - stopLineFound := false - lines := strings.Split(changelog, "\n") - for _, line := range lines { + scanner := bufio.NewScanner(strings.NewReader(changelog)) + found := false + for scanner.Scan() { + line := scanner.Text() buf = append(buf, line) - if match := stopRe.MatchString(line); match { - // o.log.Debugf("Found the stop line: %s", line) - stopLineFound = true - break - } else if matches := cveRe.FindAllString(line, -1); 0 < len(matches) { + if matches := cveRe.FindAllString(line, -1); 0 < len(matches) { for _, m := range matches { cveIDs = util.AppendIfMissing(cveIDs, m) } } + + ss := strings.Fields(line) + if len(ss) < 2 { + continue + } + + if !strings.HasPrefix(ss[1], "(") || !strings.HasSuffix(ss[1], ")") { + continue + } + clogVer, err := version.NewVersion(ss[1][1 : len(ss[1])-1]) + if err != nil { + continue + } + if installedVer.Equal(clogVer) || installedVer.GreaterThan(clogVer) { + found = true + break + } } - if !stopLineFound { - return nil, models.Changelog{ - Contents: "", - Method: models.FailedToFindVersionInChangelog, - }, fmt.Errorf( - "Failed to scan CVE IDs. The version is not in changelog. name: %s, version: %s", - name, - ver, - ) + + if !found { + pack := o.Packages[name] + pack.Changelog = models.Changelog{ + Contents: "", + Method: models.FailedToFindVersionInChangelog, + } + return nil, &pack, fmt.Errorf( + "Failed to scan CVE IDs. The version is not in changelog. name: %s, version: %s", + name, ver) } clog := models.Changelog{ - Contents: strings.Join(buf, "\n"), - Method: string(confidence.DetectionMethod), - } - - for i, p := range o.Packages { - if p.Name == name { - o.Packages[i].Changelog = clog - } + Contents: strings.Join(buf[0:len(buf)-1], "\n"), + Method: confidence.DetectionMethod, } + pack := o.Packages[name] + pack.Changelog = clog cves := []DetectedCveID{} for _, id := range cveIDs { cves = append(cves, DetectedCveID{id, confidence}) } - return cves, clog, nil + return cves, &pack, nil } func (o *debian) splitAptCachePolicy(stdout string) map[string]string { @@ -722,14 +798,3 @@ func (o *debian) parseAptCachePolicy(stdout, name string) (packCandidateVer, err } return ver, fmt.Errorf("Unknown Format: %s", stdout) } - -func appendPackIfMissing(slice []models.PackageInfo, s models.PackageInfo) []models.PackageInfo { - for _, ele := range slice { - if ele.Name == s.Name && - ele.Version == s.Version && - ele.Release == s.Release { - return slice - } - } - return append(slice, s) -} diff --git a/scan/debian_test.go b/scan/debian_test.go index f27597de..f55f094f 100644 --- a/scan/debian_test.go +++ b/scan/debian_test.go @@ -22,11 +22,11 @@ import ( "reflect" "testing" - "github.com/Sirupsen/logrus" "github.com/future-architect/vuls/cache" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/k0kubun/pp" + "github.com/sirupsen/logrus" ) func TestParseScannedPackagesLineDebian(t *testing.T) { @@ -66,7 +66,7 @@ func TestGetCveIDsFromChangelog(t *testing.T) { changelog models.Changelog }{ { - // verubuntu1 + //0 verubuntu1 []string{ "systemd", "228-4ubuntu1", @@ -81,9 +81,9 @@ systemd (228-4) unstable; urgency=medium systemd (228-3) unstable; urgency=medium`, }, []DetectedCveID{ - {"CVE-2015-2325", models.ChangelogLenientMatch}, - {"CVE-2015-2326", models.ChangelogLenientMatch}, - {"CVE-2015-3210", models.ChangelogLenientMatch}, + {"CVE-2015-2325", models.ChangelogExactMatch}, + {"CVE-2015-2326", models.ChangelogExactMatch}, + {"CVE-2015-3210", models.ChangelogExactMatch}, }, models.Changelog{ Contents: `systemd (229-2) unstable; urgency=medium @@ -92,13 +92,12 @@ systemd (228-6) unstable; urgency=medium CVE-2015-2325: heap buffer overflow in compile_branch(). (Closes: #781795) CVE-2015-2326: heap buffer overflow in pcre_compile2(). (Closes: #783285) CVE-2015-3210: heap buffer overflow in pcre_compile2() / -systemd (228-5) unstable; urgency=medium -systemd (228-4) unstable; urgency=medium`, - Method: models.ChangelogLenientMatchStr, +systemd (228-5) unstable; urgency=medium`, + Method: models.ChangelogExactMatchStr, }, }, { - // ver + //1 ver []string{ "libpcre3", "2:8.35-7.1ubuntu1", @@ -115,9 +114,9 @@ systemd (228-4) unstable; urgency=medium`, pcre3 (2:8.35-7) unstable; urgency=medium`, }, []DetectedCveID{ - {"CVE-2015-2325", models.ChangelogLenientMatch}, - {"CVE-2015-2326", models.ChangelogLenientMatch}, - {"CVE-2015-3210", models.ChangelogLenientMatch}, + {"CVE-2015-2325", models.ChangelogExactMatch}, + {"CVE-2015-2326", models.ChangelogExactMatch}, + {"CVE-2015-3210", models.ChangelogExactMatch}, }, models.Changelog{ Contents: `pcre3 (2:8.38-2) unstable; urgency=low @@ -128,13 +127,12 @@ systemd (228-4) unstable; urgency=medium`, pcre3 (2:8.35-7.2) unstable; urgency=low CVE-2015-2325: heap buffer overflow in compile_branch(). (Closes: #781795) CVE-2015-2326: heap buffer overflow in pcre_compile2(). (Closes: #783285) - CVE-2015-3210: heap buffer overflow in pcre_compile2() / - pcre3 (2:8.35-7.1) unstable; urgency=medium`, - Method: models.ChangelogLenientMatchStr, + CVE-2015-3210: heap buffer overflow in pcre_compile2() /`, + Method: models.ChangelogExactMatchStr, }, }, { - // ver-ubuntu3 + //2 ver-ubuntu3 []string{ "sysvinit", "2.88dsf-59.2ubuntu3", @@ -168,13 +166,12 @@ systemd (228-4) unstable; urgency=medium`, sysvinit (2.88dsf-59.3) unstable; urgency=medium CVE-2015-2325: heap buffer overflow in compile_branch(). (Closes: #781795) CVE-2015-2326: heap buffer overflow in pcre_compile2(). (Closes: #783285) - CVE-2015-3210: heap buffer overflow in pcre_compile2() / - sysvinit (2.88dsf-59.2ubuntu3) xenial; urgency=medium`, + CVE-2015-3210: heap buffer overflow in pcre_compile2() /`, Method: models.ChangelogExactMatchStr, }, }, { - // 1:ver-ubuntu3 + //3 1:ver-ubuntu3 []string{ "bsdutils", "1:2.27.1-1ubuntu3", @@ -192,25 +189,25 @@ systemd (228-4) unstable; urgency=medium`, util-linux (2.27-3ubuntu1) xenial; urgency=medium`, }, []DetectedCveID{ - {"CVE-2015-2325", models.ChangelogLenientMatch}, - {"CVE-2015-2326", models.ChangelogLenientMatch}, - {"CVE-2015-3210", models.ChangelogLenientMatch}, - {"CVE-2016-1000000", models.ChangelogLenientMatch}, + // {"CVE-2015-2325", models.ChangelogLenientMatch}, + // {"CVE-2015-2326", models.ChangelogLenientMatch}, + // {"CVE-2015-3210", models.ChangelogLenientMatch}, + // {"CVE-2016-1000000", models.ChangelogLenientMatch}, }, models.Changelog{ - Contents: `util-linux (2.27.1-3ubuntu1) xenial; urgency=medium - util-linux (2.27.1-3) unstable; urgency=medium - CVE-2015-2325: heap buffer overflow in compile_branch(). (Closes: #781795) - CVE-2015-2326: heap buffer overflow in pcre_compile2(). (Closes: #783285) - CVE-2015-3210: CVE-2016-1000000heap buffer overflow in pcre_compile2() / - util-linux (2.27.1-2) unstable; urgency=medium - util-linux (2.27.1-1ubuntu4) xenial; urgency=medium - util-linux (2.27.1-1ubuntu3) xenial; urgency=medium`, - Method: models.ChangelogLenientMatchStr, + // Contents: `util-linux (2.27.1-3ubuntu1) xenial; urgency=medium + // util-linux (2.27.1-3) unstable; urgency=medium + // CVE-2015-2325: heap buffer overflow in compile_branch(). (Closes: #781795) + // CVE-2015-2326: heap buffer overflow in pcre_compile2(). (Closes: #783285) + // CVE-2015-3210: CVE-2016-1000000heap buffer overflow in pcre_compile2() / + // util-linux (2.27.1-2) unstable; urgency=medium + // util-linux (2.27.1-1ubuntu4) xenial; urgency=medium + // util-linux (2.27.1-1ubuntu3) xenial; urgency=medium`, + Method: models.ChangelogExactMatchStr, }, }, { - // 1:ver-ubuntu3 + //4 1:ver-ubuntu3 []string{ "bsdutils", "1:2.27-3ubuntu3", @@ -228,29 +225,28 @@ systemd (228-4) unstable; urgency=medium`, util-linux (2.27-3) xenial; urgency=medium`, }, []DetectedCveID{ - {"CVE-2015-2325", models.ChangelogLenientMatch}, - {"CVE-2015-2326", models.ChangelogLenientMatch}, - {"CVE-2015-3210", models.ChangelogLenientMatch}, - {"CVE-2016-1000000", models.ChangelogLenientMatch}, + // {"CVE-2015-2325", models.ChangelogLenientMatch}, + // {"CVE-2015-2326", models.ChangelogLenientMatch}, + // {"CVE-2015-3210", models.ChangelogLenientMatch}, + // {"CVE-2016-1000000", models.ChangelogLenientMatch}, }, models.Changelog{ - Contents: `util-linux (2.27.1-3ubuntu1) xenial; urgency=medium - util-linux (2.27.1-3) unstable; urgency=medium - CVE-2015-2325: heap buffer overflow in compile_branch(). (Closes: #781795) - CVE-2015-2326: heap buffer overflow in pcre_compile2(). (Closes: #783285) - CVE-2015-3210: CVE-2016-1000000heap buffer overflow in pcre_compile2() / - util-linux (2.27.1-2) unstable; urgency=medium - util-linux (2.27.1-1ubuntu4) xenial; urgency=medium - util-linux (2.27.1-1ubuntu3) xenial; urgency=medium - util-linux (2.27.1-1ubuntu2) xenial; urgency=medium - util-linux (2.27.1-1ubuntu1) xenial; urgency=medium - util-linux (2.27.1-1) unstable; urgency=medium - util-linux (2.27-3) xenial; urgency=medium`, - Method: models.ChangelogLenientMatchStr, + // Contents: `util-linux (2.27.1-3ubuntu1) xenial; urgency=medium + // util-linux (2.27.1-3) unstable; urgency=medium + // CVE-2015-2325: heap buffer overflow in compile_branch(). (Closes: #781795) + // CVE-2015-2326: heap buffer overflow in pcre_compile2(). (Closes: #783285) + // CVE-2015-3210: CVE-2016-1000000heap buffer overflow in pcre_compile2() / + // util-linux (2.27.1-2) unstable; urgency=medium + // util-linux (2.27.1-1ubuntu4) xenial; urgency=medium + // util-linux (2.27.1-1ubuntu3) xenial; urgency=medium + // util-linux (2.27.1-1ubuntu2) xenial; urgency=medium + // util-linux (2.27.1-1ubuntu1) xenial; urgency=medium + // util-linux (2.27.1-1) unstable; urgency=medium`, + Method: models.ChangelogExactMatchStr, }, }, { - // https://github.com/future-architect/vuls/pull/350 + //5 https://github.com/future-architect/vuls/pull/350 []string{ "tar", "1.27.1-2+b1", @@ -259,13 +255,12 @@ systemd (228-4) unstable; urgency=medium`, tar (1.27.1-2) unstable; urgency=low`, }, []DetectedCveID{ - {"CVE-2016-6321", models.ChangelogLenientMatch}, + {"CVE-2016-6321", models.ChangelogExactMatch}, }, models.Changelog{ Contents: `tar (1.27.1-2+deb8u1) jessie-security; urgency=high - * CVE-2016-6321: Bypassing the extract path name. - tar (1.27.1-2) unstable; urgency=low`, - Method: models.ChangelogLenientMatchStr, + * CVE-2016-6321: Bypassing the extract path name.`, + Method: models.ChangelogExactMatchStr, }, }, } @@ -273,7 +268,7 @@ systemd (228-4) unstable; urgency=medium`, d := newDebian(config.ServerInfo{}) d.Distro.Family = "ubuntu" for i, tt := range tests { - aCveIDs, aClog := d.getCveIDsFromChangelog(tt.in[2], tt.in[0], tt.in[1]) + aCveIDs, aPack := d.getCveIDsFromChangelog(tt.in[2], tt.in[0], tt.in[1]) if len(aCveIDs) != len(tt.cveIDs) { t.Errorf("[%d] Len of return array are'nt same. expected %#v, actual %#v", i, tt.cveIDs, aCveIDs) t.Errorf(pp.Sprintf("%s", tt.in)) @@ -285,12 +280,12 @@ systemd (228-4) unstable; urgency=medium`, } } - if aClog.Contents != tt.changelog.Contents { - t.Errorf(pp.Sprintf("expected: %s, actual: %s", tt.changelog.Contents, aClog.Contents)) + if aPack.Changelog.Contents != tt.changelog.Contents { + t.Error(pp.Sprintf("[%d] expected: %s, actual: %s", i, tt.changelog.Contents, aPack.Changelog.Contents)) } - if aClog.Method != tt.changelog.Method { - t.Errorf(pp.Sprintf("expected: %s, actual: %s", tt.changelog.Method, aClog.Method)) + if aPack.Changelog.Method != tt.changelog.Method { + t.Error(pp.Sprintf("[%d] expected: %s, actual: %s", i, tt.changelog.Method, aPack.Changelog.Method)) } } } @@ -308,49 +303,7 @@ Reading state information... Done The following packages will be upgraded: apt ca-certificates cpio dpkg e2fslibs e2fsprogs gnupg gpgv libc-bin libc6 libcomerr2 libpcre3 libpng12-0 libss2 libssl1.0.0 libudev0 multiarch-support openssl tzdata udev upstart -21 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. -Inst dpkg [1.16.1.2ubuntu7.5] (1.16.1.2ubuntu7.7 Ubuntu:12.04/precise-updates [amd64]) -Conf dpkg (1.16.1.2ubuntu7.7 Ubuntu:12.04/precise-updates [amd64]) -Inst upstart [1.5-0ubuntu7.2] (1.5-0ubuntu7.3 Ubuntu:12.04/precise-updates [amd64]) -Inst libc-bin [2.15-0ubuntu10.10] (2.15-0ubuntu10.13 Ubuntu:12.04/precise-updates [amd64]) [libc6:amd64 ] -Conf libc-bin (2.15-0ubuntu10.13 Ubuntu:12.04/precise-updates [amd64]) [libc6:amd64 ] -Inst libc6 [2.15-0ubuntu10.10] (2.15-0ubuntu10.13 Ubuntu:12.04/precise-updates [amd64]) -Conf libc6 (2.15-0ubuntu10.13 Ubuntu:12.04/precise-updates [amd64]) -Inst libudev0 [175-0ubuntu9.9] (175-0ubuntu9.10 Ubuntu:12.04/precise-updates [amd64]) -Inst tzdata [2015a-0ubuntu0.12.04] (2015g-0ubuntu0.12.04 Ubuntu:12.04/precise-updates [all]) -Conf tzdata (2015g-0ubuntu0.12.04 Ubuntu:12.04/precise-updates [all]) -Inst e2fslibs [1.42-1ubuntu2] (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) [e2fsprogs:amd64 on e2fslibs:amd64] [e2fsprogs:amd64 ] -Conf e2fslibs (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) [e2fsprogs:amd64 ] -Inst e2fsprogs [1.42-1ubuntu2] (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) -Conf e2fsprogs (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) -Inst gpgv [1.4.11-3ubuntu2.7] (1.4.11-3ubuntu2.9 Ubuntu:12.04/precise-updates [amd64]) -Conf gpgv (1.4.11-3ubuntu2.9 Ubuntu:12.04/precise-updates [amd64]) -Inst gnupg [1.4.11-3ubuntu2.7] (1.4.11-3ubuntu2.9 Ubuntu:12.04/precise-updates [amd64]) -Conf gnupg (1.4.11-3ubuntu2.9 Ubuntu:12.04/precise-updates [amd64]) -Inst apt [0.8.16~exp12ubuntu10.22] (0.8.16~exp12ubuntu10.26 Ubuntu:12.04/precise-updates [amd64]) -Conf apt (0.8.16~exp12ubuntu10.26 Ubuntu:12.04/precise-updates [amd64]) -Inst libcomerr2 [1.42-1ubuntu2] (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) -Conf libcomerr2 (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) -Inst libss2 [1.42-1ubuntu2] (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) -Conf libss2 (1.42-1ubuntu2.3 Ubuntu:12.04/precise-updates [amd64]) -Inst libssl1.0.0 [1.0.1-4ubuntu5.21] (1.0.1-4ubuntu5.34 Ubuntu:12.04/precise-updates [amd64]) -Conf libssl1.0.0 (1.0.1-4ubuntu5.34 Ubuntu:12.04/precise-updates [amd64]) -Inst libpcre3 [8.12-4] (8.12-4ubuntu0.1 Ubuntu:12.04/precise-updates [amd64]) -Inst libpng12-0 [1.2.46-3ubuntu4] (1.2.46-3ubuntu4.2 Ubuntu:12.04/precise-updates [amd64]) -Inst multiarch-support [2.15-0ubuntu10.10] (2.15-0ubuntu10.13 Ubuntu:12.04/precise-updates [amd64]) -Conf multiarch-support (2.15-0ubuntu10.13 Ubuntu:12.04/precise-updates [amd64]) -Inst cpio [2.11-7ubuntu3.1] (2.11-7ubuntu3.2 Ubuntu:12.04/precise-updates [amd64]) -Inst udev [175-0ubuntu9.9] (175-0ubuntu9.10 Ubuntu:12.04/precise-updates [amd64]) -Inst openssl [1.0.1-4ubuntu5.33] (1.0.1-4ubuntu5.34 Ubuntu:12.04/precise-updates [amd64]) -Inst ca-certificates [20141019ubuntu0.12.04.1] (20160104ubuntu0.12.04.1 Ubuntu:12.04/precise-updates [all]) -Conf libudev0 (175-0ubuntu9.10 Ubuntu:12.04/precise-updates [amd64]) -Conf upstart (1.5-0ubuntu7.3 Ubuntu:12.04/precise-updates [amd64]) -Conf libpcre3 (8.12-4ubuntu0.1 Ubuntu:12.04/precise-updates [amd64]) -Conf libpng12-0 (1.2.46-3ubuntu4.2 Ubuntu:12.04/precise-updates [amd64]) -Conf cpio (2.11-7ubuntu3.2 Ubuntu:12.04/precise-updates [amd64]) -Conf udev (175-0ubuntu9.10 Ubuntu:12.04/precise-updates [amd64]) -Conf openssl (1.0.1-4ubuntu5.34 Ubuntu:12.04/precise-updates [amd64]) -Conf ca-certificates (20160104ubuntu0.12.04.1 Ubuntu:12.04/precise-updates [all])`, +21 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.`, []string{ "apt", "ca-certificates", @@ -391,124 +344,6 @@ The following packages will be upgraded: ntpdate passwd python3.4 python3.4-minimal rsyslog sudo sysv-rc sysvinit-utils tzdata udev util-linux 59 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. -Inst base-files [7.2ubuntu5.2] (7.2ubuntu5.4 Ubuntu:14.04/trusty-updates [amd64]) -Conf base-files (7.2ubuntu5.4 Ubuntu:14.04/trusty-updates [amd64]) -Inst coreutils [8.21-1ubuntu5.1] (8.21-1ubuntu5.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf coreutils (8.21-1ubuntu5.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst dpkg [1.17.5ubuntu5.3] (1.17.5ubuntu5.5 Ubuntu:14.04/trusty-updates [amd64]) -Conf dpkg (1.17.5ubuntu5.5 Ubuntu:14.04/trusty-updates [amd64]) -Inst libc-bin [2.19-0ubuntu6.5] (2.19-0ubuntu6.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst libc6 [2.19-0ubuntu6.5] (2.19-0ubuntu6.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst libgcc1 [1:4.9.1-0ubuntu1] (1:4.9.3-0ubuntu4 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst gcc-4.9-base [4.9.1-0ubuntu1] (4.9.3-0ubuntu4 Ubuntu:14.04/trusty-updates [amd64]) -Conf gcc-4.9-base (4.9.3-0ubuntu4 Ubuntu:14.04/trusty-updates [amd64]) -Conf libgcc1 (1:4.9.3-0ubuntu4 Ubuntu:14.04/trusty-updates [amd64]) -Conf libc6 (2.19-0ubuntu6.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf libc-bin (2.19-0ubuntu6.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst e2fslibs [1.42.9-3ubuntu1] (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) [e2fsprogs:amd64 on e2fslibs:amd64] [e2fsprogs:amd64 ] -Conf e2fslibs (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) [e2fsprogs:amd64 ] -Inst e2fsprogs [1.42.9-3ubuntu1] (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf e2fsprogs (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst login [1:4.1.5.1-1ubuntu9] (1:4.1.5.1-1ubuntu9.2 Ubuntu:14.04/trusty-updates [amd64]) -Conf login (1:4.1.5.1-1ubuntu9.2 Ubuntu:14.04/trusty-updates [amd64]) -Inst mount [2.20.1-5.1ubuntu20.4] (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf mount (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst tzdata [2015a-0ubuntu0.14.04] (2015g-0ubuntu0.14.04 Ubuntu:14.04/trusty-updates [all]) -Conf tzdata (2015g-0ubuntu0.14.04 Ubuntu:14.04/trusty-updates [all]) -Inst sysvinit-utils [2.88dsf-41ubuntu6] (2.88dsf-41ubuntu6.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst sysv-rc [2.88dsf-41ubuntu6] (2.88dsf-41ubuntu6.3 Ubuntu:14.04/trusty-updates [all]) -Conf sysv-rc (2.88dsf-41ubuntu6.3 Ubuntu:14.04/trusty-updates [all]) -Conf sysvinit-utils (2.88dsf-41ubuntu6.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst util-linux [2.20.1-5.1ubuntu20.4] (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf util-linux (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst gcc-4.8-base [4.8.2-19ubuntu1] (4.8.4-2ubuntu1~14.04.1 Ubuntu:14.04/trusty-updates [amd64]) [libstdc++6:amd64 ] -Conf gcc-4.8-base (4.8.4-2ubuntu1~14.04.1 Ubuntu:14.04/trusty-updates [amd64]) [libstdc++6:amd64 ] -Inst libstdc++6 [4.8.2-19ubuntu1] (4.8.4-2ubuntu1~14.04.1 Ubuntu:14.04/trusty-updates [amd64]) -Conf libstdc++6 (4.8.4-2ubuntu1~14.04.1 Ubuntu:14.04/trusty-updates [amd64]) -Inst libapt-pkg4.12 [1.0.1ubuntu2.6] (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Conf libapt-pkg4.12 (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Inst gpgv [1.4.16-1ubuntu2.1] (1.4.16-1ubuntu2.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf gpgv (1.4.16-1ubuntu2.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst gnupg [1.4.16-1ubuntu2.1] (1.4.16-1ubuntu2.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf gnupg (1.4.16-1ubuntu2.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst apt [1.0.1ubuntu2.6] (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Conf apt (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Inst bsdutils [1:2.20.1-5.1ubuntu20.4] (1:2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf bsdutils (1:2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst passwd [1:4.1.5.1-1ubuntu9] (1:4.1.5.1-1ubuntu9.2 Ubuntu:14.04/trusty-updates [amd64]) -Conf passwd (1:4.1.5.1-1ubuntu9.2 Ubuntu:14.04/trusty-updates [amd64]) -Inst libuuid1 [2.20.1-5.1ubuntu20.4] (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf libuuid1 (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst libblkid1 [2.20.1-5.1ubuntu20.4] (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf libblkid1 (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst libcomerr2 [1.42.9-3ubuntu1] (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf libcomerr2 (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst libmount1 [2.20.1-5.1ubuntu20.4] (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf libmount1 (2.20.1-5.1ubuntu20.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst libpcre3 [1:8.31-2ubuntu2] (1:8.31-2ubuntu2.1 Ubuntu:14.04/trusty-updates [amd64]) -Conf libpcre3 (1:8.31-2ubuntu2.1 Ubuntu:14.04/trusty-updates [amd64]) -Inst libss2 [1.42.9-3ubuntu1] (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf libss2 (1.42.9-3ubuntu1.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst libapt-inst1.5 [1.0.1ubuntu2.6] (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Inst libexpat1 [2.1.0-4ubuntu1] (2.1.0-4ubuntu1.1 Ubuntu:14.04/trusty-updates [amd64]) -Inst libffi6 [3.1~rc1+r3.0.13-12] (3.1~rc1+r3.0.13-12ubuntu0.1 Ubuntu:14.04/trusty-updates [amd64]) -Inst libgcrypt11 [1.5.3-2ubuntu4.1] (1.5.3-2ubuntu4.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst libtasn1-6 [3.4-3ubuntu0.1] (3.4-3ubuntu0.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst libgnutls-openssl27 [2.12.23-12ubuntu2.1] (2.12.23-12ubuntu2.4 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst libgnutls26 [2.12.23-12ubuntu2.1] (2.12.23-12ubuntu2.4 Ubuntu:14.04/trusty-updates [amd64]) -Inst libsqlite3-0 [3.8.2-1ubuntu2] (3.8.2-1ubuntu2.1 Ubuntu:14.04/trusty-updates [amd64]) -Inst python3.4 [3.4.0-2ubuntu1] (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst libpython3.4-stdlib [3.4.0-2ubuntu1] (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst python3.4-minimal [3.4.0-2ubuntu1] (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst libssl1.0.0 [1.0.1f-1ubuntu2.8] (1.0.1f-1ubuntu2.16 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst libpython3.4-minimal [3.4.0-2ubuntu1] (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst ntpdate [1:4.2.6.p5+dfsg-3ubuntu2.14.04.2] (1:4.2.6.p5+dfsg-3ubuntu2.14.04.8 Ubuntu:14.04/trusty-updates [amd64]) -Inst libdrm2 [2.4.56-1~ubuntu2] (2.4.64-1~ubuntu14.04.1 Ubuntu:14.04/trusty-updates [amd64]) -Inst libpng12-0 [1.2.50-1ubuntu2] (1.2.50-1ubuntu2.14.04.2 Ubuntu:14.04/trusty-updates [amd64]) -Inst initscripts [2.88dsf-41ubuntu6] (2.88dsf-41ubuntu6.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst libcgmanager0 [0.24-0ubuntu7.3] (0.24-0ubuntu7.5 Ubuntu:14.04/trusty-updates [amd64]) -Inst udev [204-5ubuntu20.10] (204-5ubuntu20.18 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst libudev1 [204-5ubuntu20.10] (204-5ubuntu20.18 Ubuntu:14.04/trusty-updates [amd64]) -Inst multiarch-support [2.19-0ubuntu6.5] (2.19-0ubuntu6.7 Ubuntu:14.04/trusty-updates [amd64]) -Conf multiarch-support (2.19-0ubuntu6.7 Ubuntu:14.04/trusty-updates [amd64]) -Inst apt-utils [1.0.1ubuntu2.6] (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Inst dh-python [1.20140128-1ubuntu8] (1.20140128-1ubuntu8.2 Ubuntu:14.04/trusty-updates [all]) -Inst iproute2 [3.12.0-2] (3.12.0-2ubuntu1 Ubuntu:14.04/trusty-updates [amd64]) -Inst ifupdown [0.7.47.2ubuntu4.1] (0.7.47.2ubuntu4.3 Ubuntu:14.04/trusty-updates [amd64]) -Inst isc-dhcp-client [4.2.4-7ubuntu12] (4.2.4-7ubuntu12.4 Ubuntu:14.04/trusty-updates [amd64]) [] -Inst isc-dhcp-common [4.2.4-7ubuntu12] (4.2.4-7ubuntu12.4 Ubuntu:14.04/trusty-updates [amd64]) -Inst rsyslog [7.4.4-1ubuntu2.5] (7.4.4-1ubuntu2.6 Ubuntu:14.04/trusty-updates [amd64]) -Inst sudo [1.8.9p5-1ubuntu1] (1.8.9p5-1ubuntu1.2 Ubuntu:14.04/trusty-updates [amd64]) -Inst cpio [2.11+dfsg-1ubuntu1.1] (2.11+dfsg-1ubuntu1.2 Ubuntu:14.04/trusty-updates [amd64]) -Conf libapt-inst1.5 (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Conf libexpat1 (2.1.0-4ubuntu1.1 Ubuntu:14.04/trusty-updates [amd64]) -Conf libffi6 (3.1~rc1+r3.0.13-12ubuntu0.1 Ubuntu:14.04/trusty-updates [amd64]) -Conf libgcrypt11 (1.5.3-2ubuntu4.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf libtasn1-6 (3.4-3ubuntu0.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf libgnutls26 (2.12.23-12ubuntu2.4 Ubuntu:14.04/trusty-updates [amd64]) -Conf libgnutls-openssl27 (2.12.23-12ubuntu2.4 Ubuntu:14.04/trusty-updates [amd64]) -Conf libsqlite3-0 (3.8.2-1ubuntu2.1 Ubuntu:14.04/trusty-updates [amd64]) -Conf libssl1.0.0 (1.0.1f-1ubuntu2.16 Ubuntu:14.04/trusty-updates [amd64]) -Conf libpython3.4-minimal (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf python3.4-minimal (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf libpython3.4-stdlib (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf python3.4 (3.4.3-1ubuntu1~14.04.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf ntpdate (1:4.2.6.p5+dfsg-3ubuntu2.14.04.8 Ubuntu:14.04/trusty-updates [amd64]) -Conf libdrm2 (2.4.64-1~ubuntu14.04.1 Ubuntu:14.04/trusty-updates [amd64]) -Conf libpng12-0 (1.2.50-1ubuntu2.14.04.2 Ubuntu:14.04/trusty-updates [amd64]) -Conf initscripts (2.88dsf-41ubuntu6.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf libcgmanager0 (0.24-0ubuntu7.5 Ubuntu:14.04/trusty-updates [amd64]) -Conf libudev1 (204-5ubuntu20.18 Ubuntu:14.04/trusty-updates [amd64]) -Conf udev (204-5ubuntu20.18 Ubuntu:14.04/trusty-updates [amd64]) -Conf apt-utils (1.0.1ubuntu2.11 Ubuntu:14.04/trusty-updates [amd64]) -Conf dh-python (1.20140128-1ubuntu8.2 Ubuntu:14.04/trusty-updates [all]) -Conf iproute2 (3.12.0-2ubuntu1 Ubuntu:14.04/trusty-updates [amd64]) -Conf ifupdown (0.7.47.2ubuntu4.3 Ubuntu:14.04/trusty-updates [amd64]) -Conf isc-dhcp-common (4.2.4-7ubuntu12.4 Ubuntu:14.04/trusty-updates [amd64]) -Conf isc-dhcp-client (4.2.4-7ubuntu12.4 Ubuntu:14.04/trusty-updates [amd64]) -Conf rsyslog (7.4.4-1ubuntu2.6 Ubuntu:14.04/trusty-updates [amd64]) -Conf sudo (1.8.9p5-1ubuntu1.2 Ubuntu:14.04/trusty-updates [amd64]) -Conf cpio (2.11+dfsg-1ubuntu1.2 Ubuntu:14.04/trusty-updates [amd64]) `, []string{ "apt", @@ -613,7 +448,7 @@ Calculating upgrade... Done func TestGetChangelogCache(t *testing.T) { const servername = "server1" - pack := models.PackageInfo{ + pack := models.Package{ Name: "apt", Version: "1.0.0", NewVersion: "1.0.1", @@ -624,7 +459,9 @@ func TestGetChangelogCache(t *testing.T) { Family: "ubuntu", Release: "16.04", }, - Packs: []models.PackageInfo{pack}, + Packs: models.Packages{ + "apt": pack, + }, } const path = "/tmp/vuls-test-cache-11111111.db" diff --git a/scan/executil.go b/scan/executil.go index 81b45452..d5aead1f 100644 --- a/scan/executil.go +++ b/scan/executil.go @@ -33,10 +33,10 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" - "github.com/Sirupsen/logrus" "github.com/cenkalti/backoff" conf "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/util" + "github.com/sirupsen/logrus" ) type execResult struct { @@ -148,6 +148,9 @@ func parallelExec(fn func(osTypeInterface) error, timeoutSec ...int) { } func exec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (result execResult) { + logger := getSSHLogger(log...) + logger.Debugf("Executing... %s", strings.Replace(cmd, "\n", "", -1)) + if c.Port == "local" && (c.Host == "127.0.0.1" || c.Host == "localhost") { result = localExec(c, cmd, sudo) @@ -157,7 +160,6 @@ func exec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (resul result = sshExecExternal(c, cmd, sudo) } - logger := getSSHLogger(log...) logger.Debug(result) return } @@ -165,7 +167,7 @@ func exec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (resul func localExec(c conf.ServerInfo, cmdstr string, sudo bool) (result execResult) { cmdstr = decorateCmd(c, cmdstr, sudo) var cmd *ex.Cmd - if c.Distro.Family == "FreeBSD" { + if c.Distro.Family == conf.FreeBSD { cmd = ex.Command("/bin/sh", "-c", cmdstr) } else { cmd = ex.Command("/bin/bash", "-c", cmdstr) diff --git a/scan/freebsd.go b/scan/freebsd.go index f8712ef6..8ac648fa 100644 --- a/scan/freebsd.go +++ b/scan/freebsd.go @@ -33,7 +33,14 @@ type bsd struct { // NewBSD constructor func newBsd(c config.ServerInfo) *bsd { - d := &bsd{} + d := &bsd{ + base: base{ + osPackages: osPackages{ + Packages: models.Packages{}, + VulnInfos: models.VulnInfos{}, + }, + }, + } d.log = util.NewCustomLogger(c) d.setServerInfo(c) return d @@ -44,13 +51,13 @@ func detectFreebsd(c config.ServerInfo) (itsMe bool, bsd osTypeInterface) { bsd = newBsd(c) // Prevent from adding `set -o pipefail` option - c.Distro = config.Distro{Family: "FreeBSD"} + c.Distro = config.Distro{Family: config.FreeBSD} if r := exec(c, "uname", noSudo); r.isSuccess() { - if strings.Contains(r.Stdout, "FreeBSD") == true { + if strings.Contains(strings.ToLower(r.Stdout), config.FreeBSD) == true { if b := exec(c, "freebsd-version", noSudo); b.isSuccess() { rel := strings.TrimSpace(b.Stdout) - bsd.setDistro("FreeBSD", rel) + bsd.setDistro(config.FreeBSD, rel) return true, bsd } } @@ -66,28 +73,54 @@ func (o *bsd) checkIfSudoNoPasswd() error { } func (o *bsd) checkDependencies() error { + o.log.Infof("Dependencies... No need") return nil } func (o *bsd) scanPackages() error { - var err error - var packs []models.PackageInfo - if packs, err = o.scanInstalledPackages(); err != nil { - o.log.Errorf("Failed to scan installed packages") + // collect the running kernel information + release, version, err := o.runningKernel() + if err != nil { + o.log.Errorf("Failed to scan the running kernel version: %s", err) return err } - o.setPackages(packs) + o.Kernel = models.Kernel{ + Release: release, + Version: version, + } - var vinfos []models.VulnInfo - if vinfos, err = o.scanUnsecurePackages(); err != nil { - o.log.Errorf("Failed to scan vulnerable packages") + rebootRequired, err := o.rebootRequired() + if err != nil { + o.log.Errorf("Failed to detect the kernel reboot required: %s", err) return err } - o.setVulnInfos(vinfos) + o.Kernel.RebootRequired = rebootRequired + + packs, err := o.scanInstalledPackages() + if err != nil { + o.log.Errorf("Failed to scan installed packages: %s", err) + return err + } + o.Packages = packs + + unsecures, err := o.scanUnsecurePackages() + if err != nil { + o.log.Errorf("Failed to scan vulnerable packages: %s", err) + return err + } + o.VulnInfos = unsecures return nil } -func (o *bsd) scanInstalledPackages() ([]models.PackageInfo, error) { +func (o *bsd) rebootRequired() (bool, error) { + r := o.exec("freebsd-version -k", noSudo) + if !r.isSuccess() { + return false, fmt.Errorf("Failed to SSH: %s", r) + } + return o.Kernel.Release != strings.TrimSpace(r.Stdout), nil +} + +func (o *bsd) scanInstalledPackages() (models.Packages, error) { cmd := util.PrependProxyEnv("pkg version -v") r := o.exec(cmd, noSudo) if !r.isSuccess() { @@ -96,7 +129,7 @@ func (o *bsd) scanInstalledPackages() ([]models.PackageInfo, error) { return o.parsePkgVersion(r.Stdout), nil } -func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) { +func (o *bsd) scanUnsecurePackages() (models.VulnInfos, error) { const vulndbPath = "/tmp/vuln.db" cmd := "rm -f " + vulndbPath r := o.exec(cmd, noSudo) @@ -111,7 +144,7 @@ func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) { } if r.ExitStatus == 0 { // no vulnerabilities - return []models.VulnInfo{}, nil + return nil, nil } var packAdtRslt []pkgAuditResult @@ -121,7 +154,7 @@ func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) { if len(cveIDs) == 0 { continue } - pack, found := o.Packages.FindByName(name) + pack, found := o.Packages[name] if !found { return nil, fmt.Errorf("Vulnerable package: %s is not found", name) } @@ -142,30 +175,38 @@ func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) { } } - for k := range cveIDAdtMap { - packs := []models.PackageInfo{} - for _, r := range cveIDAdtMap[k] { - packs = append(packs, r.pack) + vinfos := models.VulnInfos{} + for cveID := range cveIDAdtMap { + packs := models.Packages{} + for _, r := range cveIDAdtMap[cveID] { + packs[r.pack.Name] = r.pack } disAdvs := []models.DistroAdvisory{} - for _, r := range cveIDAdtMap[k] { + for _, r := range cveIDAdtMap[cveID] { disAdvs = append(disAdvs, models.DistroAdvisory{ AdvisoryID: r.vulnIDCveIDs.vulnID, }) } - vulnInfos = append(vulnInfos, models.VulnInfo{ - CveID: k, - Packages: packs, + affected := models.PackageStatuses{} + for name := range packs { + affected = append(affected, models.PackageStatus{ + Name: name, + }) + } + vinfos[cveID] = models.VulnInfo{ + CveID: cveID, + AffectedPackages: affected, DistroAdvisories: disAdvs, Confidence: models.PkgAuditMatch, - }) + } } - return + return vinfos, nil } -func (o *bsd) parsePkgVersion(stdout string) (packs []models.PackageInfo) { +func (o *bsd) parsePkgVersion(stdout string) models.Packages { + packs := models.Packages{} lines := strings.Split(stdout, "\n") for _, l := range lines { fields := strings.Fields(l) @@ -180,20 +221,26 @@ func (o *bsd) parsePkgVersion(stdout string) (packs []models.PackageInfo) { switch fields[1] { case "?", "=": - packs = append(packs, models.PackageInfo{ + packs[name] = models.Package{ Name: name, Version: ver, - }) + } case "<": candidate := strings.TrimSuffix(fields[6], ")") - packs = append(packs, models.PackageInfo{ + packs[name] = models.Package{ Name: name, Version: ver, NewVersion: candidate, - }) + } + case ">": + o.log.Warn("The installed version of the %s is newer than the current version. *This situation can arise with an out of date index file, or when testing new ports.*", name) + packs[name] = models.Package{ + Name: name, + Version: ver, + } } } - return + return packs } type vulnIDCveIDs struct { @@ -202,7 +249,7 @@ type vulnIDCveIDs struct { } type pkgAuditResult struct { - pack models.PackageInfo + pack models.Package vulnIDCveIDs vulnIDCveIDs } diff --git a/scan/freebsd_test.go b/scan/freebsd_test.go index 48dd62b1..8e1e6f96 100644 --- a/scan/freebsd_test.go +++ b/scan/freebsd_test.go @@ -12,7 +12,7 @@ import ( func TestParsePkgVersion(t *testing.T) { var tests = []struct { in string - expected []models.PackageInfo + expected models.Packages }{ { `Updating FreeBSD repository catalogue... @@ -21,27 +21,32 @@ All repositories are up-to-date. bash-4.2.45 < needs updating (remote has 4.3.42_1) gettext-0.18.3.1 < needs updating (remote has 0.19.7) tcl84-8.4.20_2,1 = up-to-date with remote +ntp-4.2.8p8_1 > succeeds port (port has 4.2.8p6) teTeX-base-3.0_25 ? orphaned: print/teTeX-base`, - []models.PackageInfo{ - { + models.Packages{ + "bash": { Name: "bash", Version: "4.2.45", NewVersion: "4.3.42_1", }, - { + "gettext": { Name: "gettext", Version: "0.18.3.1", NewVersion: "0.19.7", }, - { + "tcl84": { Name: "tcl84", Version: "8.4.20_2,1", }, - { + "teTeX-base": { Name: "teTeX-base", Version: "3.0_25", }, + "ntp": { + Name: "ntp", + Version: "4.2.8p8_1", + }, }, }, } diff --git a/scan/redhat.go b/scan/redhat.go index 4214f341..0c75f388 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -18,9 +18,9 @@ along with this program. If not, see . package scan import ( + "bufio" "fmt" "regexp" - "sort" "strings" "time" @@ -28,7 +28,7 @@ import ( "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" - "github.com/k0kubun/pp" + ver "github.com/knqyf263/go-rpm-version" ) // inherit OsTypeInterface @@ -38,7 +38,14 @@ type redhat struct { // NewRedhat is constructor func newRedhat(c config.ServerInfo) *redhat { - r := &redhat{} + r := &redhat{ + base: base{ + osPackages: osPackages{ + Packages: models.Packages{}, + VulnInfos: models.VulnInfos{}, + }, + }, + } r.log = util.NewCustomLogger(c) r.setServerInfo(c) return r @@ -49,7 +56,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { red = newRedhat(c) if r := exec(c, "ls /etc/fedora-release", noSudo); r.isSuccess() { - red.setDistro("fedora", "unknown") + red.setDistro(config.Fedora, "unknown") util.Log.Warn("Fedora not tested yet: %s", r) return true, red } @@ -66,7 +73,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { } release := result[2] - red.setDistro("oraclelinux", release) + red.setDistro(config.Oracle, release) return true, red } } @@ -87,9 +94,9 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { release := result[2] switch strings.ToLower(result[1]) { case "centos", "centos linux": - red.setDistro("centos", release) + red.setDistro(config.CentOS, release) default: - red.setDistro("rhel", release) + red.setDistro(config.RedHat, release) } return true, red } @@ -97,7 +104,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { } if r := exec(c, "ls /etc/system-release", noSudo); r.isSuccess() { - family := "amazon" + family := config.Amazon release := "unknown" if r := exec(c, "cat /etc/system-release", noSudo); r.isSuccess() { fields := strings.Fields(r.Stdout) @@ -114,7 +121,7 @@ func detectRedhat(c config.ServerInfo) (itsMe bool, red osTypeInterface) { } func (o *redhat) checkIfSudoNoPasswd() error { - if !o.sudo() { + if !config.Conf.Deep || !o.sudo() { o.log.Infof("sudo ... No need") return nil } @@ -127,12 +134,7 @@ func (o *redhat) checkIfSudoNoPasswd() error { var zero = []int{0} switch o.Distro.Family { - case "centos": - cmds = []cmd{ - {"yum --changelog --assumeno update yum", []int{0, 1}}, - } - - case "rhel", "oraclelinux": + case config.RedHat, config.Oracle: majorVersion, err := o.Distro.MajorVersion() if err != nil { return fmt.Errorf("Not implemented yet: %s, err: %s", o.Distro, err) @@ -141,14 +143,12 @@ func (o *redhat) checkIfSudoNoPasswd() error { if majorVersion < 6 { cmds = []cmd{ {"yum --color=never repolist", zero}, - {"yum --color=never check-update", []int{0, 100}}, {"yum --color=never list-security --security", zero}, {"yum --color=never info-security", zero}, } } else { cmds = []cmd{ {"yum --color=never repolist", zero}, - {"yum --color=never check-update", []int{0, 100}}, {"yum --color=never --security updateinfo list updates", zero}, {"yum --color=never --security updateinfo updates", zero}, } @@ -168,16 +168,15 @@ func (o *redhat) checkIfSudoNoPasswd() error { return nil } -// CentOS 6, 7 ... yum-plugin-changelog -// RHEL 5 ... yum-security -// RHEL 6, 7 ... - -// Amazon ... - +// - Fast scan mode +// No additional dependencies needed +// +// - Deep scan mode +// CentOS 6, 7 ... yum-utils +// RHEL 5 ... yum-security, yum-changelog +// RHEL 6, 7 ... yum-utils, yum-plugin-changelog +// Amazon ... yum-utils func (o *redhat) checkDependencies() error { - var packName string - if o.Distro.Family == "amazon" { - return nil - } - majorVersion, err := o.Distro.MajorVersion() if err != nil { msg := fmt.Sprintf("Not implemented yet: %s, err: %s", o.Distro, err) @@ -185,438 +184,481 @@ func (o *redhat) checkDependencies() error { return fmt.Errorf(msg) } - if o.Distro.Family == "centos" { + if o.Distro.Family == config.CentOS { if majorVersion < 6 { msg := fmt.Sprintf("CentOS %s is not supported", o.Distro.Release) o.log.Errorf(msg) return fmt.Errorf(msg) } + } - // --assumeno option of yum is needed. - cmd := "yum -h | grep assumeno" + packNames := []string{"yum-utils"} + if config.Conf.Deep { + switch o.Distro.Family { + case config.CentOS, config.Amazon: + packNames = append(packNames, "yum-plugin-changelog") + case config.RedHat, config.Oracle: + if majorVersion < 6 { + packNames = append(packNames, "yum-security", "yum-changelog") + } else { + packNames = append(packNames, "yum-plugin-changelog") + } + default: + return fmt.Errorf("Not implemented yet: %s", o.Distro) + } + } + + for _, name := range packNames { + cmd := "rpm -q " + name if r := o.exec(cmd, noSudo); !r.isSuccess() { - msg := fmt.Sprintf("Installed yum is old. Please update yum and then retry") + msg := fmt.Sprintf("%s is not installed", name) o.log.Errorf(msg) return fmt.Errorf(msg) } } - - switch o.Distro.Family { - case "centos": - packName = "yum-plugin-changelog" - case "rhel", "oraclelinux": - if majorVersion < 6 { - packName = "yum-security" - } else { - // yum-plugin-security is installed by default on RHEL6, 7 - return nil - } - default: - return fmt.Errorf("Not implemented yet: %s", o.Distro) - } - - cmd := "rpm -q " + packName - if r := o.exec(cmd, noSudo); !r.isSuccess() { - msg := fmt.Sprintf("%s is not installed", packName) - o.log.Errorf(msg) - return fmt.Errorf(msg) - } - o.log.Infof("Dependencies... Pass") + o.log.Infof("Dependencies ... Pass") return nil } func (o *redhat) scanPackages() error { - var err error - var packs []models.PackageInfo - if packs, err = o.scanInstalledPackages(); err != nil { - o.log.Errorf("Failed to scan installed packages") + installed, err := o.scanInstalledPackages() + if err != nil { + o.log.Errorf("Failed to scan installed packages: %s", err) return err } - o.setPackages(packs) - var vinfos []models.VulnInfo - if vinfos, err = o.scanVulnInfos(); err != nil { - o.log.Errorf("Failed to scan vulnerable packages") + rebootRequired, err := o.rebootRequired() + if err != nil { + o.log.Errorf("Failed to detect the kernel reboot required: %s", err) return err } - o.setVulnInfos(vinfos) + o.Kernel.RebootRequired = rebootRequired + + updatable, err := o.scanUpdatablePackages() + if err != nil { + o.log.Errorf("Failed to scan installed packages: %s", err) + return err + } + installed.MergeNewVersion(updatable) + o.Packages = installed + + if !config.Conf.Deep && o.Distro.Family != config.Amazon { + return nil + } + + var unsecures models.VulnInfos + if unsecures, err = o.scanUnsecurePackages(updatable); err != nil { + o.log.Errorf("Failed to scan vulnerable packages: %s", err) + return err + } + o.VulnInfos = unsecures return nil } -func (o *redhat) scanInstalledPackages() (installedPackages models.PackageInfoList, err error) { - cmd := "rpm -qa --queryformat '%{NAME}\t%{VERSION}\t%{RELEASE}\n'" - r := o.exec(cmd, noSudo) - if r.isSuccess() { - // e.g. - // openssl 1.0.1e 30.el6.11 - lines := strings.Split(r.Stdout, "\n") - for _, line := range lines { - if trimed := strings.TrimSpace(line); len(trimed) != 0 { - var packinfo models.PackageInfo - if packinfo, err = o.parseScannedPackagesLine(line); err != nil { - return - } - installedPackages = append(installedPackages, packinfo) - } - } - return +func (o *redhat) rebootRequired() (bool, error) { + r := o.exec("rpm -q --last kernel | head -n1", noSudo) + if !r.isSuccess() { + return false, fmt.Errorf("Failed to detect the last installed kernel : %v", r) } - - return installedPackages, fmt.Errorf( - "Scan packages failed. status: %d, stdout: %s, stderr: %s", - r.ExitStatus, r.Stdout, r.Stderr) + lastInstalledKernelVer := strings.Fields(r.Stdout)[0] + running := fmt.Sprintf("kernel-%s", o.Kernel.Release) + return running != lastInstalledKernelVer, nil } -func (o *redhat) parseScannedPackagesLine(line string) (models.PackageInfo, error) { - fields := strings.Fields(line) - if len(fields) != 3 { - return models.PackageInfo{}, - fmt.Errorf("Failed to parse package line: %s", line) - } - return models.PackageInfo{ - Name: fields[0], - Version: fields[1], - Release: fields[2], - }, nil -} - -func (o *redhat) scanVulnInfos() ([]models.VulnInfo, error) { - if o.Distro.Family != "centos" { - // Amazon, RHEL, Oracle Linux has yum updateinfo as default - // yum updateinfo can collenct vendor advisory information. - return o.scanUnsecurePackagesUsingYumPluginSecurity() - } - // CentOS does not have security channel... - // So, yum check-update then parse chnagelog. - return o.scanUnsecurePackagesUsingYumCheckUpdate() -} - -// For CentOS -func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, error) { - cmd := "LANGUAGE=en_US.UTF-8 yum --color=never %s check-update" - if o.getServerInfo().Enablerepo != "" { - cmd = fmt.Sprintf(cmd, "--enablerepo="+o.getServerInfo().Enablerepo) - } else { - cmd = fmt.Sprintf(cmd, "") - } - - r := o.exec(util.PrependProxyEnv(cmd), noSudo) - if !r.isSuccess(0, 100) { - //returns an exit code of 100 if there are available updates. - return nil, fmt.Errorf("Failed to SSH: %s", r) - } - - // get Updateble package name, installed, candidate version. - packInfoList, err := o.parseYumCheckUpdateLines(r.Stdout) +func (o *redhat) scanInstalledPackages() (models.Packages, error) { + release, version, err := o.runningKernel() if err != nil { - return nil, fmt.Errorf("Failed to parse %s. err: %s", cmd, err) - } - o.log.Debugf("%s", pp.Sprintf("%v", packInfoList)) - - // set candidate version info - o.Packages.MergeNewVersion(packInfoList) - - // Collect CVE-IDs in changelog - type PackInfoCveIDs struct { - PackInfo models.PackageInfo - CveIDs []string - } - - allChangelog, err := o.getAllChangelog(packInfoList) - if err != nil { - o.log.Errorf("Failed to getAllchangelog. err: %s", err) return nil, err } - - // { packageName: changelog-lines } - var rpm2changelog map[string]*string - rpm2changelog, err = o.divideChangelogByPackage(allChangelog) - if err != nil { - return nil, fmt.Errorf("Failed to parseAllChangelog. err: %s", err) + o.Kernel = models.Kernel{ + Release: release, + Version: version, } - for name, clog := range rpm2changelog { - for i, p := range o.Packages { - n := fmt.Sprintf("%s-%s-%s", - p.Name, p.NewVersion, p.NewRelease) - if name == n { - o.Packages[i].Changelog = models.Changelog{ - Contents: *clog, - Method: models.ChangelogExactMatchStr, - } - break - } - } - } - - var results []PackInfoCveIDs - for i, packInfo := range packInfoList { - changelog := o.getChangelogCVELines(rpm2changelog, packInfo) - - // Collect unique set of CVE-ID in each changelog - uniqueCveIDMap := make(map[string]bool) - lines := strings.Split(changelog, "\n") - for _, line := range lines { - cveIDs := o.parseYumUpdateinfoLineToGetCveIDs(line) - for _, c := range cveIDs { - uniqueCveIDMap[c] = true - } - } - - // keys - var cveIDs []string - for k := range uniqueCveIDMap { - cveIDs = append(cveIDs, k) - } - p := PackInfoCveIDs{ - PackInfo: packInfo, - CveIDs: cveIDs, - } - results = append(results, p) - - o.log.Infof("(%d/%d) Scanned %s-%s-%s -> %s-%s : %s", - i+1, - len(packInfoList), - p.PackInfo.Name, - p.PackInfo.Version, - p.PackInfo.Release, - p.PackInfo.NewVersion, - p.PackInfo.NewRelease, - p.CveIDs) - } - - // transform datastructure - // - From - // [ - // { - // PackInfo: models.PackageInfo, - // CveIDs: []string, - // }, - // ] - // - To - // map { - // CveID: []models.PackageInfo - // } - cveIDPackInfoMap := make(map[string][]models.PackageInfo) - for _, res := range results { - for _, cveID := range res.CveIDs { - cveIDPackInfoMap[cveID] = append( - cveIDPackInfoMap[cveID], res.PackInfo) - } - } - - vinfos := []models.VulnInfo{} - for k, v := range cveIDPackInfoMap { - // Amazon, RHEL do not use this method, so VendorAdvisory do not set. - vinfos = append(vinfos, models.VulnInfo{ - CveID: k, - Packages: v, - Confidence: models.ChangelogExactMatch, - }) - } - return vinfos, nil -} - -// parseYumCheckUpdateLines parse yum check-update to get package name, candidate version -func (o *redhat) parseYumCheckUpdateLines(stdout string) (results models.PackageInfoList, err error) { - needToParse := false - lines := strings.Split(stdout, "\n") - for _, line := range lines { - // update information of packages begin after blank line. - if trimed := strings.TrimSpace(line); len(trimed) == 0 { - needToParse = true - continue - } - if needToParse { - if strings.HasPrefix(line, "Obsoleting") || - strings.HasPrefix(line, "Security:") { - // see https://github.com/future-architect/vuls/issues/165 - continue - } - candidate, err := o.parseYumCheckUpdateLine(line) - if err != nil { - return results, err - } - - installed, found := o.Packages.FindByName(candidate.Name) - if !found { - o.log.Warnf("Not found the package in rpm -qa. candidate: %s-%s-%s", - candidate.Name, candidate.Version, candidate.Release) - results = append(results, candidate) - continue - } - installed.NewVersion = candidate.NewVersion - installed.NewRelease = candidate.NewRelease - installed.Repository = candidate.Repository - results = append(results, installed) - } - } - return -} - -func (o *redhat) parseYumCheckUpdateLine(line string) (models.PackageInfo, error) { - fields := strings.Fields(line) - if len(fields) < 3 { - return models.PackageInfo{}, fmt.Errorf("Unknown format: %s", line) - } - splitted := strings.Split(fields[0], ".") - packName := "" - if len(splitted) == 1 { - packName = fields[0] + installed := models.Packages{} + var cmd string + majorVersion, _ := o.Distro.MajorVersion() + if majorVersion < 6 { + cmd = "rpm -qa --queryformat '%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{ARCH}\n'" } else { - packName = strings.Join(strings.Split(fields[0], ".")[0:(len(splitted)-1)], ".") + cmd = "rpm -qa --queryformat '%{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE} %{ARCH}\n'" + } + r := o.exec(cmd, noSudo) + if !r.isSuccess() { + return nil, fmt.Errorf("Scan packages failed: %s", r) } - verfields := strings.Split(fields[1], "-") - if len(verfields) != 2 { - return models.PackageInfo{}, fmt.Errorf("Unknown format: %s", line) - } - version := o.regexpReplace(verfields[0], `^[0-9]+:`, "") - release := verfields[1] - repos := strings.Join(fields[2:len(fields)], " ") - - return models.PackageInfo{ - Name: packName, - NewVersion: version, - NewRelease: release, - Repository: repos, - }, nil -} - -func (o *redhat) mkPstring() *string { - str := "" - return &str -} - -func (o *redhat) regexpReplace(src string, pat string, rep string) string { - re := regexp.MustCompile(pat) - return re.ReplaceAllString(src, rep) -} - -var changeLogCVEPattern = regexp.MustCompile(`CVE-[0-9]+-[0-9]+`) - -func (o *redhat) getChangelogCVELines(rpm2changelog map[string]*string, packInfo models.PackageInfo) string { - rpm := fmt.Sprintf("%s-%s-%s", packInfo.Name, packInfo.NewVersion, packInfo.NewRelease) - retLine := "" - if rpm2changelog[rpm] != nil { - lines := strings.Split(*rpm2changelog[rpm], "\n") - for _, line := range lines { - if changeLogCVEPattern.MatchString(line) { - retLine += fmt.Sprintf("%s\n", line) - } - } - } - return retLine -} - -func (o *redhat) divideChangelogByPackage(allChangelog string) (map[string]*string, error) { - var majorVersion int - var err error - if o.Distro.Family == "centos" { - majorVersion, err = o.Distro.MajorVersion() - if err != nil { - return nil, fmt.Errorf("Not implemented yet: %s, err: %s", o.Distro, err) - } - } - - orglines := strings.Split(allChangelog, "\n") - tmpline := "" - var lines []string - var prev, now bool - for i := range orglines { - if majorVersion == 5 { - /* for CentOS5 (yum-util < 1.1.20) */ - prev = false - now = false - if 0 < i { - prev, err = o.isRpmPackageNameLine(orglines[i-1]) - if err != nil { - return nil, err - } - } - now, err = o.isRpmPackageNameLine(orglines[i]) + // openssl 0 1.0.1e 30.el6.11 x86_64 + lines := strings.Split(r.Stdout, "\n") + for _, line := range lines { + if trimed := strings.TrimSpace(line); len(trimed) != 0 { + pack, err := o.parseInstalledPackagesLine(line) if err != nil { return nil, err } - if prev && now { - tmpline = fmt.Sprintf("%s, %s", tmpline, orglines[i]) - continue - } - if !prev && now { - tmpline = fmt.Sprintf("%s%s", tmpline, orglines[i]) - continue - } - if tmpline != "" { - lines = append(lines, fmt.Sprintf("%s", tmpline)) - tmpline = "" - } - lines = append(lines, fmt.Sprintf("%s", orglines[i])) - } else { - /* for CentOS6,7 (yum-util >= 1.1.20) */ - line := orglines[i] - line = o.regexpReplace(line, `^ChangeLog for: `, "") - line = o.regexpReplace(line, `^\*\*\sNo\sChangeLog\sfor:.*`, "") - lines = append(lines, line) - } - } - rpm2changelog := make(map[string]*string) - writePointer := o.mkPstring() - for _, line := range lines { - match, err := o.isRpmPackageNameLine(line) - if err != nil { - return nil, err - } - if match { - rpms := strings.Split(line, ",") - pNewString := o.mkPstring() - writePointer = pNewString - for _, rpm := range rpms { - rpm = strings.TrimSpace(rpm) - rpm = o.regexpReplace(rpm, `^[0-9]+:`, "") - rpm = o.regexpReplace(rpm, `\.(i386|i486|i586|i686|k6|athlon|x86_64|noarch|ppc|alpha|sparc)$`, "") - rpm2changelog[rpm] = pNewString + // Kernel package may be isntalled multiple versions. + // From the viewpoint of vulnerability detection, + // pay attention only to the running kernel + if pack.Name == "kernel" { + ver := fmt.Sprintf("%s-%s.%s", pack.Version, pack.Release, pack.Arch) + if o.Kernel.Release != ver { + o.log.Debugf("Not a running kernel: %s, uname: %s", ver, release) + continue + } else { + o.log.Debugf("Running kernel: %s, uname: %s", ver, release) + } } - } else { - if strings.HasPrefix(line, "Dependencies Resolved") { - return rpm2changelog, nil - } - *writePointer += fmt.Sprintf("%s\n", line) + installed[pack.Name] = pack } } - return rpm2changelog, nil + return installed, nil } -// CentOS -func (o *redhat) getAllChangelog(packInfoList models.PackageInfoList) (stdout string, err error) { - packageNames := "" - for _, packInfo := range packInfoList { - packageNames += fmt.Sprintf("%s ", packInfo.Name) +func (o *redhat) parseInstalledPackagesLine(line string) (models.Package, error) { + fields := strings.Fields(line) + if len(fields) != 5 { + return models.Package{}, + fmt.Errorf("Failed to parse package line: %s", line) + } + ver := "" + epoch := fields[1] + if epoch == "0" || epoch == "(none)" { + ver = fields[2] + } else { + ver = fmt.Sprintf("%s:%s", epoch, fields[2]) } - command := "" - if 0 < len(config.Conf.HTTPProxy) { - command += util.ProxyEnv() + return models.Package{ + Name: fields[0], + Version: ver, + Release: fields[3], + Arch: fields[4], + }, nil +} + +func (o *redhat) scanUpdatablePackages() (models.Packages, error) { + cmd := "repoquery --all --pkgnarrow=updates --qf='%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{REPO}'" + for _, repo := range o.getServerInfo().Enablerepo { + cmd += " --enablerepo=" + repo } + r := o.exec(util.PrependProxyEnv(cmd), o.sudo()) + if !r.isSuccess() { + return nil, fmt.Errorf("Failed to SSH: %s", r) + } + + // Collect Updateble packages, installed, candidate version and repository. + return o.parseUpdatablePacksLines(r.Stdout) +} + +// parseUpdatablePacksLines parse the stdout of repoquery to get package name, candidate version +func (o *redhat) parseUpdatablePacksLines(stdout string) (models.Packages, error) { + updatable := models.Packages{} + lines := strings.Split(stdout, "\n") + for _, line := range lines { + // TODO remove + // if strings.HasPrefix(line, "Obsoleting") || + // strings.HasPrefix(line, "Security:") { + // // see https://github.com/future-architect/vuls/issues/165 + // continue + // } + if len(strings.TrimSpace(line)) == 0 { + continue + } + pack, err := o.parseUpdatablePacksLine(line) + if err != nil { + return updatable, err + } + updatable[pack.Name] = pack + } + return updatable, nil +} + +func (o *redhat) parseUpdatablePacksLine(line string) (models.Package, error) { + fields := strings.Fields(line) + if len(fields) < 5 { + return models.Package{}, fmt.Errorf("Unknown format: %s, fields: %s", line, fields) + } + + ver := "" + epoch := fields[1] + if epoch == "0" { + ver = fields[2] + } else { + ver = fmt.Sprintf("%s:%s", epoch, fields[2]) + } + + repos := strings.Join(fields[4:len(fields)], " ") + + p := models.Package{ + Name: fields[0], + NewVersion: ver, + NewRelease: fields[3], + Repository: repos, + } + return p, nil +} + +func (o *redhat) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) { + if config.Conf.Deep { + //TODO Cache changelogs to bolt + if err := o.fillChangelogs(updatable); err != nil { + return nil, err + } + } + + if o.Distro.Family != config.CentOS { + // Amazon, RHEL, Oracle Linux has yum updateinfo as default + // yum updateinfo can collenct vendor advisory information. + return o.scanCveIDsByCommands(updatable) + } + + // Parse chnagelog because CentOS does not have security channel... + return o.scanCveIDsInChangelog(updatable) +} + +func (o *redhat) fillChangelogs(updatables models.Packages) error { + names := []string{} + for name := range updatables { + names = append(names, name) + } + + if err := o.fillDiffChangelogs(names); err != nil { + return err + } + + emptyChangelogPackNames := []string{} + for _, pack := range o.Packages { + if pack.NewVersion != "" && pack.Changelog.Contents == "" { + emptyChangelogPackNames = append(emptyChangelogPackNames, pack.Name) + } + } + + i := 0 + for _, name := range emptyChangelogPackNames { + i++ + o.log.Infof("(%d/%d) Fetched Changelogs %s", i, len(emptyChangelogPackNames), name) + if err := o.fillDiffChangelogs([]string{name}); err != nil { + return err + } + } + + return nil +} + +func (o *redhat) getAvailableChangelogs(packNames []string) (map[string]string, error) { yumopts := "" - if o.getServerInfo().Enablerepo != "" { - yumopts = " --enablerepo=" + o.getServerInfo().Enablerepo + if 0 < len(o.getServerInfo().Enablerepo) { + yumopts = " --enablerepo=" + strings.Join(o.getServerInfo().Enablerepo, ",") } if config.Conf.SkipBroken { yumopts += " --skip-broken" } + cmd := `yum --color=never %s changelog all %s | grep -A 10000 '==================== Available Packages ===================='` + cmd = fmt.Sprintf(cmd, yumopts, strings.Join(packNames, " ")) - // yum update --changelog doesn't have --color option. - command += fmt.Sprintf(" LANGUAGE=en_US.UTF-8 yum --changelog --assumeno update %s ", yumopts) + packageNames - - r := o.exec(command, sudo) + r := o.exec(util.PrependProxyEnv(cmd), o.sudo()) if !r.isSuccess(0, 1) { - return "", fmt.Errorf( - "Failed to get changelog. status: %d, stdout: %s, stderr: %s", - r.ExitStatus, r.Stdout, r.Stderr) + return nil, fmt.Errorf("Failed to SSH: %s", r) } - return strings.Replace(r.Stdout, "\r", "", -1), nil + + return o.divideChangelogsIntoEachPackages(r.Stdout), nil +} + +// Divide available change logs of all updatable packages into each package's changelog +func (o *redhat) divideChangelogsIntoEachPackages(stdout string) map[string]string { + changelogs := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(stdout)) + + crlf, newBlock := false, true + packNameVer, contents := "", []string{} + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "==================== Available Packages ====================") { + continue + } + if newBlock { + left := strings.Fields(line)[0] + // ss := strings.Split(left, ".") + // packNameVer = strings.Join(ss[0:len(ss)-1], ".") + packNameVer = left + newBlock = false + continue + } + if len(strings.TrimSpace(line)) == 0 { + if crlf { + changelogs[packNameVer] = strings.Join(contents, "\n") + packNameVer = "" + contents = []string{} + newBlock = true + crlf = false + } else { + contents = append(contents, line) + crlf = true + } + } else { + contents = append(contents, line) + crlf = false + } + } + if 0 < len(contents) { + changelogs[packNameVer] = strings.Join(contents, "\n") + } + return changelogs +} + +func (o *redhat) fillDiffChangelogs(packNames []string) error { + changelogs, err := o.getAvailableChangelogs(packNames) + if err != nil { + return err + } + + for s := range changelogs { + // name, pack, found := o.Packages.FindOne(func(p models.Package) bool { + name, pack, found := o.Packages.FindOne(func(p models.Package) bool { + var epochNameVerRel string + if index := strings.Index(p.NewVersion, ":"); 0 < index { + epoch := p.NewVersion[0:index] + ver := p.NewVersion[index+1 : len(p.NewVersion)] + epochNameVerRel = fmt.Sprintf("%s:%s-%s", + epoch, p.Name, ver) + } else { + epochNameVerRel = fmt.Sprintf("%s-%s", + p.Name, p.NewVersion) + } + return strings.HasPrefix(s, epochNameVerRel) + }) + + if found { + diff, err := o.getDiffChangelog(pack, changelogs[s]) + detectionMethod := models.ChangelogExactMatchStr + + if err != nil { + o.log.Debug(err) + // Try without epoch + if index := strings.Index(pack.Version, ":"); 0 < index { + pack.Version = pack.Version[index+1 : len(pack.Version)] + o.log.Debug("Try without epoch", pack) + diff, err = o.getDiffChangelog(pack, changelogs[s]) + if err != nil { + o.log.Debugf("Failed to find the version in changelog: %s-%s-%s", + pack.Name, pack.Version, pack.Release) + detectionMethod = models.FailedToFindVersionInChangelog + } else { + o.log.Debugf("Found the version in changelog without epoch: %s-%s-%s", + pack.Name, pack.Version, pack.Release) + detectionMethod = models.ChangelogLenientMatchStr + } + } + } + + pack = o.Packages[name] + pack.Changelog = models.Changelog{ + Contents: diff, + Method: models.DetectionMethod(detectionMethod), + } + o.Packages[name] = pack + } + } + return nil +} + +func (o *redhat) getDiffChangelog(pack models.Package, availableChangelog string) (string, error) { + installedVer := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) + scanner := bufio.NewScanner(strings.NewReader(availableChangelog)) + diff := []string{} + found := false + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "* ") { + diff = append(diff, line) + continue + } + + // openssh on RHEL + // openssh-server-6.6.1p1-35.el7_3.x86_64 rhui-rhel-7-server-rhui-rpms + // Wed Mar 1 21:00:00 2017 Jakub Jelen - 6.6.1p1-35 + 0.9.3-9 + ss := strings.Split(line, " + ") + if 1 < len(ss) { + line = ss[0] + } + + ss = strings.Split(line, " ") + if len(ss) < 2 { + diff = append(diff, line) + continue + } + v := ss[len(ss)-1] + v = strings.TrimPrefix(v, "-") + v = strings.TrimPrefix(v, "[") + v = strings.TrimSuffix(v, "]") + + // On Amazon often end with email address. Go to next line + if strings.HasPrefix(v, "<") && strings.HasSuffix(v, ">") { + diff = append(diff, line) + continue + } + + version := ver.NewVersion(v) + if installedVer.Equal(version) || installedVer.GreaterThan(version) { + found = true + break + } + diff = append(diff, line) + } + + if len(diff) == 0 || !found { + return availableChangelog, + fmt.Errorf("Failed to find the version in changelog: %s-%s-%s", + pack.Name, pack.Version, pack.Release) + } + return strings.TrimSpace(strings.Join(diff, "\n")), nil +} + +func (o *redhat) scanCveIDsInChangelog(updatable models.Packages) (models.VulnInfos, error) { + packCveIDs := make(map[string][]string) + for name := range updatable { + cveIDs := []string{} + pack := o.Packages[name] + if pack.Changelog.Method == models.FailedToFindVersionInChangelog { + continue + } + scanner := bufio.NewScanner(strings.NewReader(pack.Changelog.Contents)) + for scanner.Scan() { + if matches := cveRe.FindAllString(scanner.Text(), -1); 0 < len(matches) { + for _, m := range matches { + cveIDs = util.AppendIfMissing(cveIDs, m) + } + } + } + packCveIDs[name] = cveIDs + } + + // transform datastructure + // - From + // "packname": []{"CVE-2017-1111", ".../ + // + // - To + // map { + // "CVE-2017-1111": "packname", + // } + vinfos := models.VulnInfos{} + for name, cveIDs := range packCveIDs { + for _, cid := range cveIDs { + if v, ok := vinfos[cid]; ok { + v.AffectedPackages = append(v.AffectedPackages, models.PackageStatus{Name: name}) + vinfos[cid] = v + } else { + vinfos[cid] = models.VulnInfo{ + CveID: cid, + AffectedPackages: models.PackageStatuses{{Name: name}}, + Confidence: models.ChangelogExactMatch, + } + } + } + } + return vinfos, nil } type distroAdvisoryCveIDs struct { @@ -626,10 +668,9 @@ type distroAdvisoryCveIDs struct { // Scaning unsecure packages using yum-plugin-security. // Amazon, RHEL, Oracle Linux -func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, error) { - if o.Distro.Family == "centos" { +func (o *redhat) scanCveIDsByCommands(updatable models.Packages) (models.VulnInfos, error) { + if o.Distro.Family == config.CentOS { // CentOS has no security channel. - // So use yum check-update && parse changelog return nil, fmt.Errorf( "yum updateinfo is not suppported on CentOS") } @@ -646,7 +687,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, return nil, fmt.Errorf("Not implemented yet: %s, err: %s", o.Distro, err) } - if (o.Distro.Family == "rhel" || o.Distro.Family == "oraclelinux") && major == 5 { + if (o.Distro.Family == config.RedHat || o.Distro.Family == config.Oracle) && major == 5 { cmd = "yum --color=never list-security --security" } else { cmd = "yum --color=never --security updateinfo list updates" @@ -657,39 +698,23 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, } advIDPackNamesList, err := o.parseYumUpdateinfoListAvailable(r.Stdout) - // get package name, version, rel to be upgrade. - cmd = "LANGUAGE=en_US.UTF-8 yum --color=never check-update" - r = o.exec(util.PrependProxyEnv(cmd), o.sudo()) - if !r.isSuccess(0, 100) { - //returns an exit code of 100 if there are available updates. - return nil, fmt.Errorf("Failed to SSH: %s", r) - } - updatable, err := o.parseYumCheckUpdateLines(r.Stdout) - if err != nil { - return nil, fmt.Errorf("Failed to parse %s. err: %s", cmd, err) - } - o.log.Debugf("%s", pp.Sprintf("%v", updatable)) - - // set candidate version info - o.Packages.MergeNewVersion(updatable) - - dict := map[string][]models.PackageInfo{} + dict := make(map[string]models.Packages) for _, advIDPackNames := range advIDPackNamesList { - packInfoList := models.PackageInfoList{} + packages := models.Packages{} for _, packName := range advIDPackNames.PackNames { - packInfo, found := updatable.FindByName(packName) + pack, found := updatable[packName] if !found { return nil, fmt.Errorf( - "PackInfo not found. packInfo: %#v", packName) + "Package not found. pack: %#v", packName) } - packInfoList = append(packInfoList, packInfo) + packages[pack.Name] = pack continue } - dict[advIDPackNames.AdvisoryID] = packInfoList + dict[advIDPackNames.AdvisoryID] = packages } // get advisoryID(RHSA, ALAS, ELSA) - CVE IDs - if (o.Distro.Family == "rhel" || o.Distro.Family == "oraclelinux") && major == 5 { + if (o.Distro.Family == config.RedHat || o.Distro.Family == config.Oracle) && major == 5 { cmd = "yum --color=never info-security" } else { cmd = "yum --color=never --security updateinfo updates" @@ -702,36 +727,36 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, if err != nil { return nil, err } - // pp.Println(advisoryCveIDsList) // All information collected. // Convert to VulnInfos. vinfos := models.VulnInfos{} for _, advIDCveIDs := range advisoryCveIDsList { for _, cveID := range advIDCveIDs.CveIDs { - found := false - for i, p := range vinfos { - if cveID == p.CveID { - advAppended := append(p.DistroAdvisories, advIDCveIDs.DistroAdvisory) - vinfos[i].DistroAdvisories = advAppended + vinfo, found := vinfos[cveID] + if found { + advAppended := append(vinfo.DistroAdvisories, advIDCveIDs.DistroAdvisory) + vinfo.DistroAdvisories = advAppended - packs := dict[advIDCveIDs.DistroAdvisory.AdvisoryID] - vinfos[i].Packages = append(vinfos[i].Packages, packs...) - found = true - break + packs := dict[advIDCveIDs.DistroAdvisory.AdvisoryID] + for _, pack := range packs { + vinfo.AffectedPackages = append(vinfo.AffectedPackages, + models.PackageStatus{Name: pack.Name}) } - } - - if !found { - cpinfo := models.VulnInfo{ + } else { + packs := dict[advIDCveIDs.DistroAdvisory.AdvisoryID] + affected := models.PackageStatuses{} + for _, p := range packs { + affected = append(affected, models.PackageStatus{Name: p.Name}) + } + vinfo = models.VulnInfo{ CveID: cveID, DistroAdvisories: []models.DistroAdvisory{advIDCveIDs.DistroAdvisory}, - Packages: dict[advIDCveIDs.DistroAdvisory.AdvisoryID], + AffectedPackages: affected, Confidence: models.YumUpdateSecurityMatch, } - vinfos = append(vinfos, cpinfo) } - + vinfos[cveID] = vinfo } } return vinfos, nil @@ -750,22 +775,19 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID cveIDsSetInThisSection := make(map[string]bool) // use this flag to Collect CVE IDs in CVEs field. - var inDesctiption = false + inDesctiption, inCves := false, false for _, line := range lines { line = strings.TrimSpace(line) // find the new section pattern if horizontalRulePattern.MatchString(line) { - // set previous section's result to return-variable if sectionState == Content { - foundCveIDs := []string{} for cveID := range cveIDsSetInThisSection { foundCveIDs = append(foundCveIDs, cveID) } - sort.Strings(foundCveIDs) result = append(result, distroAdvisoryCveIDs{ DistroAdvisory: advisory, CveIDs: foundCveIDs, @@ -773,7 +795,8 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID // reset for next section. cveIDsSetInThisSection = make(map[string]bool) - inDesctiption = false + inDesctiption, inCves = false, false + advisory = models.DistroAdvisory{} } // Go to next section @@ -784,49 +807,72 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID switch sectionState { case Header: switch o.Distro.Family { - case "centos": + case config.CentOS: // CentOS has no security channel. - // So use yum check-update && parse changelog return result, fmt.Errorf( "yum updateinfo is not suppported on CentOS") - case "rhel", "amazon", "oraclelinux": + case config.RedHat, config.Amazon, config.Oracle: // nop } case Content: if found := o.isDescriptionLine(line); found { - inDesctiption = true + inDesctiption, inCves = true, false + ss := strings.Split(line, " : ") + advisory.Description += fmt.Sprintf("%s\n", + strings.Join(ss[1:len(ss)], " : ")) + continue } // severity - severity, found := o.parseYumUpdateinfoToGetSeverity(line) - if found { + if severity, found := o.parseYumUpdateinfoToGetSeverity(line); found { advisory.Severity = severity + continue } // No need to parse in description except severity if inDesctiption { + if ss := strings.Split(line, ": "); 1 < len(ss) { + advisory.Description += fmt.Sprintf("%s\n", + strings.Join(ss[1:len(ss)], ": ")) + } continue } - cveIDs := o.parseYumUpdateinfoLineToGetCveIDs(line) - for _, cveID := range cveIDs { - cveIDsSetInThisSection[cveID] = true + if found := o.isCvesHeaderLine(line); found { + inCves = true + ss := strings.Split(line, "CVEs : ") + line = strings.Join(ss[1:len(ss)], " ") + cveIDs := o.parseYumUpdateinfoLineToGetCveIDs(line) + for _, cveID := range cveIDs { + cveIDsSetInThisSection[cveID] = true + } + continue + } + + if inCves { + cveIDs := o.parseYumUpdateinfoLineToGetCveIDs(line) + for _, cveID := range cveIDs { + cveIDsSetInThisSection[cveID] = true + } } advisoryID, found := o.parseYumUpdateinfoToGetAdvisoryID(line) if found { advisory.AdvisoryID = advisoryID + continue } issued, found := o.parseYumUpdateinfoLineToGetIssued(line) if found { advisory.Issued = issued + continue } updated, found := o.parseYumUpdateinfoLineToGetUpdated(line) if found { advisory.Updated = updated + continue } } } @@ -850,22 +896,8 @@ func (o *redhat) changeSectionState(state int) (newState int) { return newState } -var rpmPackageArchPattern = regexp.MustCompile( - `^[^ ]+\.(i386|i486|i586|i686|k6|athlon|x86_64|noarch|ppc|alpha|sparc)$`) - -func (o *redhat) isRpmPackageNameLine(line string) (bool, error) { - s := strings.TrimPrefix(line, "ChangeLog for: ") - ss := strings.Split(s, ", ") - if len(ss) == 0 { - return false, nil - } - for _, s := range ss { - s = strings.TrimRight(s, " \r\n") - if !rpmPackageArchPattern.MatchString(s) { - return false, nil - } - } - return true, nil +func (o *redhat) isCvesHeaderLine(line string) bool { + return strings.Contains(line, "CVEs : ") } var yumCveIDPattern = regexp.MustCompile(`(CVE-\d{4}-\d{4,})`) @@ -999,9 +1031,10 @@ func (o *redhat) clone() osTypeInterface { func (o *redhat) sudo() bool { switch o.Distro.Family { - case "amazon": + case config.Amazon, config.CentOS: return false default: - return true + // RHEL, Oracle + return config.Conf.Deep } } diff --git a/scan/redhat_test.go b/scan/redhat_test.go index 16097892..9b59f1a4 100644 --- a/scan/redhat_test.go +++ b/scan/redhat_test.go @@ -6,9 +6,7 @@ it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License @@ -19,6 +17,8 @@ package scan import ( "reflect" + "sort" + "strings" "testing" "time" @@ -38,28 +38,28 @@ func TestParseScanedPackagesLineRedhat(t *testing.T) { var packagetests = []struct { in string - pack models.PackageInfo + pack models.Package }{ { - "openssl 1.0.1e 30.el6.11", - models.PackageInfo{ + "openssl 0 1.0.1e 30.el6.11 x86_64", + models.Package{ Name: "openssl", Version: "1.0.1e", Release: "30.el6.11", }, }, { - "Percona-Server-shared-56 5.6.19 rel67.0.el6", - models.PackageInfo{ + "Percona-Server-shared-56 1 5.6.19 rel67.0.el6 x84_64", + models.Package{ Name: "Percona-Server-shared-56", - Version: "5.6.19", + Version: "1:5.6.19", Release: "rel67.0.el6", }, }, } for _, tt := range packagetests { - p, _ := r.parseScannedPackagesLine(tt.in) + p, _ := r.parseInstalledPackagesLine(tt.in) if p.Name != tt.pack.Name { t.Errorf("name: expected %s, actual %s", tt.pack.Name, p.Name) } @@ -261,54 +261,6 @@ func TestIsDescriptionLine(t *testing.T) { } } -func TestIsRpmPackageNameLine(t *testing.T) { - r := newRedhat(config.ServerInfo{}) - var tests = []struct { - in string - found bool - }{ - { - "stunnel-4.15-2.el5.2.i386", - true, - }, - { - "iproute-2.6.18-15.el5.i386", - true, - }, - { - "1:yum-updatesd-0.9-6.el5_10.noarch", - true, - }, - { - "glibc-2.12-1.192.el6.x86_64", - true, - }, - { - " glibc-2.12-1.192.el6.x86_64", - false, - }, - { - "glibc-2.12-1.192.el6.x86_64, iproute-2.6.18-15.el5.i386", - true, - }, - { - "k6 hoge.i386", - false, - }, - { - "triathlon", - false, - }, - } - - for i, tt := range tests { - found, err := r.isRpmPackageNameLine(tt.in) - if tt.found != found { - t.Errorf("[%d] line: %s, expected %t, actual %t, err %v", i, tt.in, tt.found, found, err) - } - } -} - func TestParseYumUpdateinfoToGetSeverity(t *testing.T) { r := newRedhat(config.ServerInfo{}) var tests = []struct { @@ -355,11 +307,8 @@ func TestParseYumUpdateinfoOL(t *testing.T) { Issued : 2017-02-15 CVEs : CVE-2017-3135 Description : [32:9.9.4-38.2] - : - Fix CVE-2017-3135 (ISC change 4557) - : - Fix and test caching CNAME before DNAME (ISC - : change 4558) Severity : Moderate - + =============================================================================== openssl security update =============================================================================== @@ -371,12 +320,8 @@ Description : [32:9.9.4-38.2] CVEs : CVE-2016-8610 : CVE-2017-3731 Description : [1.0.1e-48.4] - : - fix CVE-2017-3731 - DoS via truncated packets - : with RC4-MD5 cipher - : - fix CVE-2016-8610 - DoS of single-threaded - : servers via excessive alerts Severity : Moderate - + =============================================================================== Unbreakable Enterprise kernel security update =============================================================================== @@ -387,10 +332,6 @@ Description : [1.0.1e-48.4] Issued : 2017-02-15 CVEs : CVE-2017-6074 Description : kernel-uek - : [4.1.12-61.1.28] - : - dccp: fix freeing skb too early for - : IPV6_RECVPKTINFO (Andrey Konovalov) [Orabug: - : 25598257] {CVE-2017-6074} Severity : Important ` @@ -408,17 +349,19 @@ Description : kernel-uek []distroAdvisoryCveIDs{ { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "ELSA-2017-0276", - Severity: "Moderate", - Issued: issued, + AdvisoryID: "ELSA-2017-0276", + Severity: "Moderate", + Issued: issued, + Description: "[32:9.9.4-38.2]\n", }, CveIDs: []string{"CVE-2017-3135"}, }, { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "ELSA-2017-0286", - Severity: "Moderate", - Issued: issued, + AdvisoryID: "ELSA-2017-0286", + Severity: "Moderate", + Issued: issued, + Description: "[1.0.1e-48.4]\n", }, CveIDs: []string{ "CVE-2016-8610", @@ -427,9 +370,10 @@ Description : kernel-uek }, { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "ELSA-2017-3520", - Severity: "Important", - Issued: issued, + AdvisoryID: "ELSA-2017-3520", + Severity: "Important", + Issued: issued, + Description: "kernel-uek\n", }, CveIDs: []string{"CVE-2017-6074"}, }, @@ -439,11 +383,15 @@ Description : kernel-uek for _, tt := range tests { actual, _ := r.parseYumUpdateinfo(tt.in) for i, advisoryCveIDs := range actual { - if !reflect.DeepEqual(tt.out[i], advisoryCveIDs) { - e := pp.Sprintf("%v", tt.out[i]) - a := pp.Sprintf("%v", advisoryCveIDs) + if tt.out[i].DistroAdvisory != advisoryCveIDs.DistroAdvisory { t.Errorf("[%d] Alas is not same. \nexpected: %s\nactual: %s", - i, e, a) + i, tt.out[i].DistroAdvisory, advisoryCveIDs.DistroAdvisory) + } + sort.Strings(tt.out[i].CveIDs) + sort.Strings(advisoryCveIDs.CveIDs) + if !reflect.DeepEqual(tt.out[i].CveIDs, advisoryCveIDs.CveIDs) { + t.Errorf("[%d] Alas is not same. \nexpected: %s\nactual: %s", + i, tt.out[i].CveIDs, advisoryCveIDs.CveIDs) } } } @@ -462,12 +410,6 @@ func TestParseYumUpdateinfoRHEL(t *testing.T) { Bugs : 1259087 - CVE-2015-5722 bind: malformed DNSSEC key failed assertion denial of service CVEs : CVE-2015-5722 Description : The Berkeley Internet Name Domain (BIND) is an implementation of - : the Domain Name System (DNS) protocols. BIND - : includes a DNS server (named); a resolver library - : (routines for applications to use when interfacing - : with DNS); and tools for verifying that the DNS - : server is operating correctly. - : Severity : Important =============================================================================== @@ -483,12 +425,6 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of CVEs : CVE-2015-8000 : CVE-2015-8001 Description : The Berkeley Internet Name Domain (BIND) is an implementation of - : the Domain Name System (DNS) protocols. BIND - : includes a DNS server (named); a resolver library - : (routines for applications to use when interfacing - : with DNS); and tools for verifying that the DNS - : server is operating correctly. - : Severity : Low =============================================================================== @@ -499,17 +435,25 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of Type : security Status : final Issued : 2015-09-03 02:00:00 - Bugs : 1299364 - CVE-2015-8704 bind: specific APL data could trigger an INSIST in apl_42.c CVEs : CVE-2015-8704 + Bugs : 1299364 - CVE-2015-8704 bind: specific APL data could trigger an INSIST in apl_42.c + CVEs : CVE-2015-8704 : CVE-2015-8705 Description : The Berkeley Internet Name Domain (BIND) is an implementation of - : the Domain Name System (DNS) protocols. BIND - : includes a DNS server (named); a resolver library - : (routines for applications to use when interfacing - : with DNS); and tools for verifying that the DNS - : server is operating correctly. - : + : CVE-2015-10000 Severity : Moderate +=============================================================================== + Moderate: sudo security update +=============================================================================== + Update ID : RHSA-2017:1574 + Release : 0 + Type : security + Status : final + Issued : 2015-09-03 02:00:00 + Bugs : 1459152 - CVE-2017-1000368 sudo: Privilege escalation via improper get_process_ttyname() parsing (insufficient fix for CVE-2017-1000367) CVEs : CVE-2017-1000368 +Description : The sudo packages contain the sudo utility which allows system + : administrators to provide certain users with the + Severity : Moderate ` issued, _ := time.Parse("2006-01-02", "2015-09-03") updated, _ := time.Parse("2006-01-02", "2015-09-04") @@ -526,18 +470,20 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of []distroAdvisoryCveIDs{ { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "RHSA-2015:1705", - Severity: "Important", - Issued: issued, + AdvisoryID: "RHSA-2015:1705", + Severity: "Important", + Issued: issued, + Description: "The Berkeley Internet Name Domain (BIND) is an implementation of\n", }, CveIDs: []string{"CVE-2015-5722"}, }, { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "RHSA-2015:2655", - Severity: "Low", - Issued: issued, - Updated: updated, + AdvisoryID: "RHSA-2015:2655", + Severity: "Low", + Issued: issued, + Updated: updated, + Description: "The Berkeley Internet Name Domain (BIND) is an implementation of\n", }, CveIDs: []string{ "CVE-2015-8000", @@ -546,26 +492,39 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of }, { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "RHSA-2016:0073", - Severity: "Moderate", - Issued: issued, - Updated: updated, + AdvisoryID: "RHSA-2016:0073", + Severity: "Moderate", + Issued: issued, + Description: "The Berkeley Internet Name Domain (BIND) is an implementation of\nCVE-2015-10000\n", }, CveIDs: []string{ "CVE-2015-8704", "CVE-2015-8705", }, }, + { + DistroAdvisory: models.DistroAdvisory{ + AdvisoryID: "RHSA-2017:1574", + Severity: "Moderate", + Issued: issued, + Description: "The sudo packages contain the sudo utility which allows system\nadministrators to provide certain users with the\n", + }, + CveIDs: []string{ + "CVE-2017-1000368", + }, + }, }, }, } for _, tt := range tests { actual, _ := r.parseYumUpdateinfo(tt.in) for i, advisoryCveIDs := range actual { + sort.Strings(tt.out[i].CveIDs) + sort.Strings(advisoryCveIDs.CveIDs) if !reflect.DeepEqual(tt.out[i], advisoryCveIDs) { e := pp.Sprintf("%v", tt.out[i]) a := pp.Sprintf("%v", advisoryCveIDs) - t.Errorf("[%d] Alas is not same. \nexpected: %s\nactual: %s", + t.Errorf("[%d] not same. \nexpected: %s\nactual: %s", i, e, a) } } @@ -578,7 +537,7 @@ func TestParseYumUpdateinfoAmazon(t *testing.T) { r.Distro = config.Distro{Family: "redhat"} issued, _ := time.Parse("2006-01-02", "2015-12-15") - updated, _ := time.Parse("2006-01-02", "2015-12-16") + // updated, _ := time.Parse("2006-01-02", "2015-12-16") var tests = []struct { in string @@ -595,10 +554,8 @@ func TestParseYumUpdateinfoAmazon(t *testing.T) { Issued : 2015-12-15 13:30 CVEs : CVE-2016-1494 Description : Package updates are available for Amazon Linux AMI that fix the - : following vulnerabilities: CVE-2016-1494: - : 1295869: - : CVE-2016-1494 python-rsa: Signature forgery using - : Bleichenbacher'06 attack + : CVE-20160-1111 + : hogehoge Severity : medium =============================================================================== @@ -613,32 +570,26 @@ Description : Package updates are available for Amazon Linux AMI that fix the : CVE-2015-3195 : CVE-2015-3196 Description : Package updates are available for Amazon Linux AMI that fix the - : following vulnerabilities: CVE-2015-3196: - : 1288326: - : CVE-2015-3196 OpenSSL: Race condition handling PSK - : identify hint A race condition flaw, leading to a - : double free, was found in the way OpenSSL handled - : pre-shared keys (PSKs). A remote attacker could - : use this flaw to crash a multi-threaded SSL/TLS - : client. - : + : foo bar baz + : hoge fuga hega Severity : medium`, []distroAdvisoryCveIDs{ { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "ALAS-2016-644", - Severity: "medium", - Issued: issued, + AdvisoryID: "ALAS-2016-644", + Severity: "medium", + Issued: issued, + Description: "Package updates are available for Amazon Linux AMI that fix the\nCVE-20160-1111\nhogehoge\n", }, CveIDs: []string{"CVE-2016-1494"}, }, { DistroAdvisory: models.DistroAdvisory{ - AdvisoryID: "ALAS-2015-614", - Severity: "medium", - Issued: issued, - Updated: updated, + AdvisoryID: "ALAS-2015-614", + Severity: "medium", + Issued: issued, + Description: "Package updates are available for Amazon Linux AMI that fix the\nfoo bar baz\nhoge fuga hega\n", }, CveIDs: []string{ "CVE-2015-3194", @@ -653,6 +604,8 @@ Description : Package updates are available for Amazon Linux AMI that fix the for _, tt := range tests { actual, _ := r.parseYumUpdateinfo(tt.in) for i, advisoryCveIDs := range actual { + sort.Strings(tt.out[i].CveIDs) + sort.Strings(actual[i].CveIDs) if !reflect.DeepEqual(tt.out[i], advisoryCveIDs) { e := pp.Sprintf("%v", tt.out[i]) a := pp.Sprintf("%v", advisoryCveIDs) @@ -663,128 +616,123 @@ Description : Package updates are available for Amazon Linux AMI that fix the } } -func TestParseYumCheckUpdateLines(t *testing.T) { +func TestParseYumCheckUpdateLine(t *testing.T) { r := newRedhat(config.ServerInfo{}) r.Distro = config.Distro{Family: "centos"} - stdout := `Loaded plugins: changelog, fastestmirror, keys, protect-packages, protectbase, security -Loading mirror speeds from cached hostfile - * base: mirror.fairway.ne.jp - * epel: epel.mirror.srv.co.ge - * extras: mirror.fairway.ne.jp - * updates: mirror.fairway.ne.jp -0 packages excluded due to repository protections - -audit-libs.x86_64 2.3.7-5.el6 base -bash.x86_64 4.1.2-33.el6_7.1 updates -Obsoleting Packages -python-libs.i686 2.6.6-64.el6 rhui-REGION-rhel-server-releases - python-ordereddict.noarch 1.1-3.el6ev installed -bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates -pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 -` - - r.Packages = []models.PackageInfo{ - { - Name: "audit-libs", - Version: "2.3.6", - Release: "4.el6", - }, - { - Name: "bash", - Version: "4.1.1", - Release: "33", - }, - { - Name: "python-libs", - Version: "2.6.0", - Release: "1.1-0", - }, - { - Name: "python-ordereddict", - Version: "1.0", - Release: "1", - }, - { - Name: "bind-utils", - Version: "1.0", - Release: "1", - }, - { - Name: "pytalloc", - Version: "2.0.1", - Release: "0", - }, - } var tests = []struct { in string - out models.PackageInfoList + out models.Package }{ { - stdout, - models.PackageInfoList{ - { - Name: "audit-libs", - Version: "2.3.6", - Release: "4.el6", - NewVersion: "2.3.7", - NewRelease: "5.el6", - Repository: "base", - }, - { - Name: "bash", - Version: "4.1.1", - Release: "33", - NewVersion: "4.1.2", - NewRelease: "33.el6_7.1", - Repository: "updates", - }, - { - Name: "python-libs", - Version: "2.6.0", - Release: "1.1-0", - NewVersion: "2.6.6", - NewRelease: "64.el6", - Repository: "rhui-REGION-rhel-server-releases", - }, - { - Name: "python-ordereddict", - Version: "1.0", - Release: "1", - NewVersion: "1.1", - NewRelease: "3.el6ev", - Repository: "installed", - }, - { - Name: "bind-utils", - Version: "1.0", - Release: "1", - NewVersion: "9.3.6", - NewRelease: "25.P1.el5_11.8", - Repository: "updates", - }, - { - Name: "pytalloc", - Version: "2.0.1", - Release: "0", - NewVersion: "2.0.7", - NewRelease: "2.el6", - Repository: "@CentOS 6.5/6.5", - }, + "zlib 0 1.2.7 17.el7 rhui-REGION-rhel-server-releases", + models.Package{ + Name: "zlib", + NewVersion: "1.2.7", + NewRelease: "17.el7", + Repository: "rhui-REGION-rhel-server-releases", + }, + }, + { + "shadow-utils 2 4.1.5.1 24.el7 rhui-REGION-rhel-server-releases", + models.Package{ + Name: "shadow-utils", + NewVersion: "2:4.1.5.1", + NewRelease: "24.el7", + Repository: "rhui-REGION-rhel-server-releases", }, }, } for _, tt := range tests { - packInfoList, err := r.parseYumCheckUpdateLines(tt.in) + aPack, err := r.parseUpdatablePacksLine(tt.in) if err != nil { t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in) return } - for i, ePackInfo := range tt.out { - if !reflect.DeepEqual(ePackInfo, packInfoList[i]) { - e := pp.Sprintf("%v", ePackInfo) - a := pp.Sprintf("%v", packInfoList[i]) - t.Errorf("[%d] expected %s, actual %s", i, e, a) + if !reflect.DeepEqual(tt.out, aPack) { + e := pp.Sprintf("%v", tt.out) + a := pp.Sprintf("%v", aPack) + t.Errorf("expected %s, actual %s", e, a) + } + } +} + +func TestParseYumCheckUpdateLines(t *testing.T) { + r := newRedhat(config.ServerInfo{}) + r.Distro = config.Distro{Family: "centos"} + stdout := `audit-libs 0 2.3.7 5.el6 base +bash 0 4.1.2 33.el6_7.1 updates +python-libs 0 2.6.6 64.el6 rhui-REGION-rhel-server-releases +python-ordereddict 0 1.1 3.el6ev installed +bind-utils 30 9.3.6 25.P1.el5_11.8 updates +pytalloc 0 2.0.7 2.el6 @CentOS 6.5/6.5` + + r.Packages = models.NewPackages( + models.Package{Name: "audit-libs"}, + models.Package{Name: "bash"}, + models.Package{Name: "python-libs"}, + models.Package{Name: "python-ordereddict"}, + models.Package{Name: "bind-utils"}, + models.Package{Name: "pytalloc"}, + ) + var tests = []struct { + in string + out models.Packages + }{ + { + stdout, + models.NewPackages( + models.Package{ + Name: "audit-libs", + NewVersion: "2.3.7", + NewRelease: "5.el6", + Repository: "base", + }, + models.Package{ + Name: "bash", + NewVersion: "4.1.2", + NewRelease: "33.el6_7.1", + Repository: "updates", + }, + models.Package{ + Name: "python-libs", + NewVersion: "2.6.6", + NewRelease: "64.el6", + Repository: "rhui-REGION-rhel-server-releases", + }, + models.Package{ + Name: "python-ordereddict", + NewVersion: "1.1", + NewRelease: "3.el6ev", + Repository: "installed", + }, + models.Package{ + Name: "bind-utils", + NewVersion: "30:9.3.6", + NewRelease: "25.P1.el5_11.8", + Repository: "updates", + }, + models.Package{ + Name: "pytalloc", + NewVersion: "2.0.7", + NewRelease: "2.el6", + Repository: "@CentOS 6.5/6.5", + }, + ), + }, + } + + for _, tt := range tests { + packages, err := r.parseUpdatablePacksLines(tt.in) + if err != nil { + t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in) + return + } + for name, ePack := range tt.out { + if !reflect.DeepEqual(ePack, packages[name]) { + e := pp.Sprintf("%v", ePack) + a := pp.Sprintf("%v", packages[name]) + t.Errorf("expected %s, actual %s", e, a) } } } @@ -793,76 +741,54 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 func TestParseYumCheckUpdateLinesAmazon(t *testing.T) { r := newRedhat(config.ServerInfo{}) r.Distro = config.Distro{Family: "amazon"} - stdout := `Loaded plugins: priorities, update-motd, upgrade-helper -34 package(s) needed for security, out of 71 available - -bind-libs.x86_64 32:9.8.2-0.37.rc1.45.amzn1 amzn-main -java-1.7.0-openjdk.x86_64 1.7.0.95-2.6.4.0.65.amzn1 amzn-main -if-not-architecture 100-200 amzn-main -` - r.Packages = []models.PackageInfo{ - { - Name: "bind-libs", - Version: "9.8.0", - Release: "0.33.rc1.45.amzn1", - }, - { - Name: "java-1.7.0-openjdk", - Version: "1.7.0.0", - Release: "2.6.4.0.0.amzn1", - }, - { - Name: "if-not-architecture", - Version: "10", - Release: "20", - }, - } + stdout := `bind-libs 32 9.8.2 0.37.rc1.45.amzn1 amzn-main +java-1.7.0-openjdk 0 1.7.0.95 2.6.4.0.65.amzn1 amzn-main +if-not-architecture 0 100 200 amzn-main` + r.Packages = models.NewPackages( + models.Package{Name: "bind-libs"}, + models.Package{Name: "java-1.7.0-openjdk"}, + models.Package{Name: "if-not-architecture"}, + ) var tests = []struct { in string - out models.PackageInfoList + out models.Packages }{ { stdout, - models.PackageInfoList{ - { + models.NewPackages( + models.Package{ Name: "bind-libs", - Version: "9.8.0", - Release: "0.33.rc1.45.amzn1", - NewVersion: "9.8.2", + NewVersion: "32:9.8.2", NewRelease: "0.37.rc1.45.amzn1", Repository: "amzn-main", }, - { + models.Package{ Name: "java-1.7.0-openjdk", - Version: "1.7.0.0", - Release: "2.6.4.0.0.amzn1", NewVersion: "1.7.0.95", NewRelease: "2.6.4.0.65.amzn1", Repository: "amzn-main", }, - { + models.Package{ Name: "if-not-architecture", - Version: "10", - Release: "20", NewVersion: "100", NewRelease: "200", Repository: "amzn-main", }, - }, + ), }, } for _, tt := range tests { - packInfoList, err := r.parseYumCheckUpdateLines(tt.in) + packages, err := r.parseUpdatablePacksLines(tt.in) if err != nil { t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in) return } - for i, ePackInfo := range tt.out { - if !reflect.DeepEqual(ePackInfo, packInfoList[i]) { - e := pp.Sprintf("%v", ePackInfo) - a := pp.Sprintf("%v", packInfoList[i]) - t.Errorf("[%d] expected %s, actual %s", i, e, a) + for name, ePack := range tt.out { + if !reflect.DeepEqual(ePack, packages[name]) { + e := pp.Sprintf("%v", ePack) + a := pp.Sprintf("%v", packages[name]) + t.Errorf("[%s] expected %s, actual %s", name, e, a) } } } @@ -967,308 +893,407 @@ func TestExtractPackNameVerRel(t *testing.T) { } -const ( - /* for CentOS6,7 (yum-util >= 1.1.20) */ - stdoutCentos6 = `---> Package libaio.x86_64 0:0.3.107-10.el6 will be installed ---> Finished Dependency Resolution - -Changes in packages about to be updated: - -ChangeLog for: binutils-2.20.51.0.2-5.44.el6.x86_64 -* Mon Dec 7 21:00:00 2015 Nick Clifton - 2.20.51.0.2-5.44 -- Backport upstream RELRO fixes. (#1227839) - -** No ChangeLog for: chkconfig-1.3.49.5-1.el6.x86_64 - -ChangeLog for: coreutils-8.4-43.el6.x86_64, coreutils-libs-8.4-43.el6.x86_64 -* Wed Feb 10 21:00:00 2016 Ondrej Vasik - 8.4-43 -- sed should actually be /bin/sed (related #1222140) - -* Wed Jan 6 21:00:00 2016 Ondrej Vasik - 8.4-41 -- colorls.sh,colorls.csh - call utilities with complete path (#1222140) -- mkdir, mkfifo, mknod - respect default umask/acls when - COREUTILS_CHILD_DEFAULT_ACLS envvar is set (to match rhel 7 behaviour, - -ChangeLog for: centos-release-6-8.el6.centos.12.3.x86_64 -* Wed May 18 21:00:00 2016 Johnny Hughes 6-8.el6.centos.12.3 -- CentOS-6.8 Released -- TESTSTRING CVE-0000-0000 - -ChangeLog for: 12:dhclient-4.1.1-51.P1.el6.centos.x86_64, 12:dhcp-common-4.1.1-51.P1.el6.centos.x86_64 -* Tue May 10 21:00:00 2016 Johnny Hughes - 12:4.1.1-51.P1 -- created patch 1000 for CentOS Branding -- replaced vvendor variable with CentOS in the SPEC file -- TESTSTRING CVE-1111-1111 - -* Mon Jan 11 21:00:00 2016 Jiri Popelka - 12:4.1.1-51.P1 -- send unicast request/release via correct interface (#1297445) - -* Thu Dec 3 21:00:00 2015 Jiri Popelka - 12:4.1.1-50.P1 -- Lease table overflow crash. (#1133917) -- Add ignore-client-uids option. (#1196768) -- dhclient-script: it's OK if the arping reply comes from our system. (#1204095) -- VLAN ID is only bottom 12-bits of TCI. (#1259552) -- dhclient: Make sure link-local address is ready in stateless mode. (#1263466) -- dhclient-script: make_resolv_conf(): Keep old nameservers - if server sends domain-name/search, but no nameservers. (#1269595) - -ChangeLog for: file-5.04-30.el6.x86_64, file-libs-5.04-30.el6.x86_64 -* Tue Feb 16 21:00:00 2016 Jan Kaluza 5.04-30 -- fix CVE-2014-3538 (unrestricted regular expression matching) - -* Tue Jan 5 21:00:00 2016 Jan Kaluza 5.04-29 -- fix #1284826 - try to read ELF header to detect corrupted one - -* Wed Dec 16 21:00:00 2015 Jan Kaluza 5.04-28 -- fix #1263987 - fix bugs found by coverity in the patch - -* Thu Nov 26 21:00:00 2015 Jan Kaluza 5.04-27 -- fix CVE-2014-3587 (incomplete fix for CVE-2012-1571) -- fix CVE-2014-3710 (out-of-bounds read in elf note headers) -- fix CVE-2014-8116 (multiple DoS issues (resource consumption)) -- fix CVE-2014-8117 (denial of service issue (resource consumption)) -- fix CVE-2014-9620 (limit the number of ELF notes processed) -- fix CVE-2014-9653 (malformed elf file causes access to uninitialized memory) - - -Dependencies Resolved - -` - /* for CentOS5 (yum-util < 1.1.20) */ - stdoutCentos5 = `---> Package portmap.i386 0:4.0-65.2.2.1 set to be updated ---> Finished Dependency Resolution - -Changes in packages about to be updated: - -libuser-0.54.7-3.el5.i386 -nss_db-2.2-38.el5_11.i386 -* Thu Nov 20 23:00:00 2014 Nalin Dahyabhai - 2.2-38 -- build without strict aliasing (internal build tooling) - -* Sat Nov 15 23:00:00 2014 Nalin Dahyabhai - 2.2-37 -- pull in fix for a memory leak in nss_db (#1163493) - -acpid-1.0.4-12.el5.i386 -* Thu Oct 6 00:00:00 2011 Jiri Skala - 1.0.4-12 -- Resolves: #729769 - acpid dumping useless info to log - -nash-5.1.19.6-82.el5.i386, mkinitrd-5.1.19.6-82.el5.i386 -* Tue Apr 15 00:00:00 2014 Brian C. Lane 5.1.19.6-82 -- Use ! instead of / when searching sysfs for ccis device - Resolves: rhbz#988020 -- Always include ahci module (except on s390) (bcl) - Resolves: rhbz#978245 -- Prompt to recreate default initrd (karsten) - Resolves: rhbz#472764 - -util-linux-2.13-0.59.el5_8.i386 -* Wed Oct 17 00:00:00 2012 Karel Zak 2.13-0.59.el5_8 -- fix #865791 - fdisk fails to partition disk not in use - -* Wed Dec 21 23:00:00 2011 Karel Zak 2.13-0.59 -- fix #768382 - CVE-2011-1675 CVE-2011-1677 util-linux various flaws - -* Wed Oct 26 00:00:00 2011 Karel Zak 2.13-0.58 -- fix #677452 - util-linux fails to build with gettext-0.17 - -30:bind-utils-9.3.6-25.P1.el5_11.8.i386, 30:bind-libs-9.3.6-25.P1.el5_11.8.i386 -* Mon Mar 14 23:00:00 2016 Tomas Hozza - 30:9.3.6-25.P1.8 -- Fix issue with patch for CVE-2016-1285 and CVE-2016-1286 found by test suite - -* Wed Mar 9 23:00:00 2016 Tomas Hozza - 30:9.3.6-25.P1.7 -- Fix CVE-2016-1285 and CVE-2016-1286 - -* Mon Jan 18 23:00:00 2016 Tomas Hozza - 30:9.3.6-25.P1.6 -- Fix CVE-2015-8704 - -* Thu Sep 3 00:00:00 2015 Tomas Hozza - 30:9.3.6-25.P1.5 -- Fix CVE-2015-8000 - - -Dependencies Resolved - -` -) - -func TestGetChangelogCVELines(t *testing.T) { - var testsCentos6 = []struct { - in models.PackageInfo - out string - }{ - { - models.PackageInfo{ - Name: "binutils", - NewVersion: "2.20.51.0.2", - NewRelease: "5.44.el6", - }, - "", - }, - { - models.PackageInfo{ - Name: "centos-release", - NewVersion: "6", - NewRelease: "8.el6.centos.12.3", - }, - `- TESTSTRING CVE-0000-0000 -`, - }, - { - models.PackageInfo{ - Name: "dhclient", - NewVersion: "4.1.1", - NewRelease: "51.P1.el6.centos", - }, - `- TESTSTRING CVE-1111-1111 -`, - }, - { - models.PackageInfo{ - Name: "dhcp-common", - NewVersion: "4.1.1", - NewRelease: "51.P1.el6.centos", - }, - `- TESTSTRING CVE-1111-1111 -`, - }, - { - models.PackageInfo{ - Name: "coreutils-libs", - NewVersion: "8.4", - NewRelease: "43.el6", - }, - "", - }, - { - models.PackageInfo{ - Name: "file", - NewVersion: "5.04", - NewRelease: "30.el6", - }, - `- fix CVE-2014-3538 (unrestricted regular expression matching) -- fix CVE-2014-3587 (incomplete fix for CVE-2012-1571) -- fix CVE-2014-3710 (out-of-bounds read in elf note headers) -- fix CVE-2014-8116 (multiple DoS issues (resource consumption)) -- fix CVE-2014-8117 (denial of service issue (resource consumption)) -- fix CVE-2014-9620 (limit the number of ELF notes processed) -- fix CVE-2014-9653 (malformed elf file causes access to uninitialized memory) -`, - }, - { - models.PackageInfo{ - Name: "file-libs", - NewVersion: "5.04", - NewRelease: "30.el6", - }, - `- fix CVE-2014-3538 (unrestricted regular expression matching) -- fix CVE-2014-3587 (incomplete fix for CVE-2012-1571) -- fix CVE-2014-3710 (out-of-bounds read in elf note headers) -- fix CVE-2014-8116 (multiple DoS issues (resource consumption)) -- fix CVE-2014-8117 (denial of service issue (resource consumption)) -- fix CVE-2014-9620 (limit the number of ELF notes processed) -- fix CVE-2014-9653 (malformed elf file causes access to uninitialized memory) -`, - }, - } - +func TestGetDiffChangelog(t *testing.T) { r := newRedhat(config.ServerInfo{}) - r.Distro = config.Distro{ - Family: "centos", - Release: "6.7", - } - for _, tt := range testsCentos6 { - rpm2changelog, err := r.divideChangelogByPackage(stdoutCentos6) - if err != nil { - t.Errorf("err: %s", err) - } - changelog := r.getChangelogCVELines(rpm2changelog, tt.in) - if tt.out != changelog { - t.Errorf("line: expected %s, actual %s, tt: %#v", tt.out, changelog, tt) - } + type in struct { + pack models.Package + changelog string } - var testsCentos5 = []struct { - in models.PackageInfo + var tests = []struct { + in in out string }{ + // 0 { - models.PackageInfo{ - Name: "libuser", - NewVersion: "0.54.7", - NewRelease: "3.el5", + in: in{ + pack: models.Package{ + Version: "2017a", + Release: "1", + }, + changelog: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Thu Mar 2 12:00:00 2017 Patsy Franklin - 2017a-1 +- Rebase to tzdata-2017a + - Mongolia no longer observes DST. (BZ #1425222) + - Add upstream patch to fix over-runing of POSIX limit on zone abbreviations. +- Add zone1970.tab file to the install list. (BZ #1427694) + +* Wed Nov 23 12:00:00 2016 Patsy Franklin - 2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04.`, }, - "", + out: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017.`, }, + // 1 { - models.PackageInfo{ - Name: "nss_db", - NewVersion: "2.2", - NewRelease: "38.el5_11", + in: in{ + pack: models.Package{ + Version: "2004e", + Release: "2", + }, + changelog: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Wed Nov 23 12:00:00 2016 Patsy Franklin - 2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04. + +* Mon Nov 29 12:00:00 2004 Jakub Jelinek 2004g-1 +- 2004g (#141107) +- updates for Cuba + +* Mon Oct 11 12:00:00 2004 Jakub Jelinek 2004e-2 +- 2004e (#135194) +- updates for Brazil, Uruguay and Argentina`, }, - "", + out: `* Mon Mar 20 12:00:00 2017 Patsy Franklin - 2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Wed Nov 23 12:00:00 2016 Patsy Franklin - 2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04. + +* Mon Nov 29 12:00:00 2004 Jakub Jelinek 2004g-1 +- 2004g (#141107) +- updates for Cuba`, }, + // 2 { - models.PackageInfo{ - Name: "acpid", - NewVersion: "1.0.4", - NewRelease: "82.el5", + in: in{ + pack: models.Package{ + Version: "2016j", + Release: "1", + }, + changelog: `* Mon Mar 20 12:00:00 2017 Patsy Franklin -2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017. + +* Wed Nov 23 12:00:00 2016 Patsy Franklin -2016j-1 +- Rebase to tzdata-2016ij + - Saratov region of Russia is moving from +03 offset to +04 offset + on 2016-12-04.`, }, - "", + out: `* Mon Mar 20 12:00:00 2017 Patsy Franklin -2017b-1 +- Rebase to tzdata-2017b. + - Haiti resumed DST on March 12, 2017.`, }, + // 3 { - models.PackageInfo{ - Name: "mkinitrd", - NewVersion: "5.1.19.6", - NewRelease: "82.el5", + in: in{ + pack: models.Package{ + Version: "3.10.0", + Release: "327.22.1.el7", + }, + changelog: `* Thu Jun 9 21:00:00 2016 Alexander Gordeev [3.10.0-327.22.2.el7] +- [infiniband] security: Restrict use of the write() interface (Don Dutile) [1332553 1316685] {CVE-2016-4565} + +* Mon May 16 21:00:00 2016 Alexander Gordeev [3.10.0-327.22.1.el7] +- [mm] mmu_notifier: fix memory corruption (Jerome Glisse) [1335727 1307042] +- [misc] cxl: Increase timeout for detection of AFU mmio hang (Steve Best) [1335419 1329682] +- [misc] cxl: Configure the PSL for two CAPI ports on POWER8NVL (Steve Best) [1336389 1278793]`, }, - "", + out: `* Thu Jun 9 21:00:00 2016 Alexander Gordeev [3.10.0-327.22.2.el7] +- [infiniband] security: Restrict use of the write() interface (Don Dutile) [1332553 1316685] {CVE-2016-4565}`, }, + // 4 { - models.PackageInfo{ - Name: "util-linux", - NewVersion: "2.13", - NewRelease: "0.59.el5_8", + in: in{ + pack: models.Package{ + Version: "6.6.1p1", + Release: "34", + }, + + changelog: `* Wed Mar 1 21:00:00 2017 Jakub Jelen - 6.6.1p1-35 + 0.9.3-9 +- Do not send SD_NOTIFY from forked childern (#1381997) + +* Fri Feb 24 21:00:00 2017 Jakub Jelen - 6.6.1p1-34 + 0.9.3-9 +- Add SD_NOTIFY code to help systemd to track running service (#1381997)`, }, - `- fix #768382 - CVE-2011-1675 CVE-2011-1677 util-linux various flaws -`, + out: `* Wed Mar 1 21:00:00 2017 Jakub Jelen - 6.6.1p1-35 +- Do not send SD_NOTIFY from forked childern (#1381997)`, }, + // 5 { - models.PackageInfo{ - Name: "bind-libs", - NewVersion: "9.3.6", - NewRelease: "25.P1.el5_11.8", + in: in{ + pack: models.Package{ + Version: "2.1.23", + Release: "15.el6", + }, + changelog: `* Fri Feb 27 12:00:00 2015 Jakub Jelen 2.1.23-15.2 +- Support AIX SASL GSSAPI (#1174315) + +* Tue Nov 18 12:00:00 2014 Petr Lautrbach 2.1.23-15.1 +- check a context value in sasl_gss_encode() (#1087221) + +* Mon Jun 23 12:00:00 2014 Petr Lautrbach 2.1.23-15 +- don't use " for saslauth user's description (#1081445) +- backport the ad_compat option (#994242) +- fixed a memory leak in the client side DIGEST-MD5 code (#838628)`, }, - `- Fix issue with patch for CVE-2016-1285 and CVE-2016-1286 found by test suite -- Fix CVE-2016-1285 and CVE-2016-1286 -- Fix CVE-2015-8704 -- Fix CVE-2015-8000 -`, + out: `* Fri Feb 27 12:00:00 2015 Jakub Jelen 2.1.23-15.2 +- Support AIX SASL GSSAPI (#1174315) + +* Tue Nov 18 12:00:00 2014 Petr Lautrbach 2.1.23-15.1 +- check a context value in sasl_gss_encode() (#1087221)`, }, + // 6 { - models.PackageInfo{ - Name: "bind-utils", - NewVersion: "9.3.6", - NewRelease: "25.P1.el5_11.8", + in: in{ + pack: models.Package{ + Version: "3.6.20", + Release: "1.el6", + }, + changelog: `* Wed Jul 29 12:00:00 2015 Jan Stanek - 3.6.20-1.2 +- Add patch for compiler warnings highlighted by rpmdiff. + Related: rhbz#1244727 + +* Wed Jul 22 12:00:00 2015 Viktor Jancik - 3.6.20-1.el6_7.1 +- fix for CVE-2015-3416 + Resolves: #1244727 + +* Tue Nov 17 12:00:00 2009 Panu Matilainen - 3.6.20-1 +- update to 3.6.20 (http://www.sqlite.org/releaselog/3_6_20.html) + +* Tue Oct 6 12:00:00 2009 Panu Matilainen - 3.6.18-1 +- update to 3.6.18 (http://www.sqlite.org/releaselog/3_6_18.html) +- drop no longer needed test-disabler patches`, }, - `- Fix issue with patch for CVE-2016-1285 and CVE-2016-1286 found by test suite -- Fix CVE-2016-1285 and CVE-2016-1286 -- Fix CVE-2015-8704 -- Fix CVE-2015-8000 -`, + out: `* Wed Jul 29 12:00:00 2015 Jan Stanek - 3.6.20-1.2 +- Add patch for compiler warnings highlighted by rpmdiff. + Related: rhbz#1244727 + +* Wed Jul 22 12:00:00 2015 Viktor Jancik - 3.6.20-1.el6_7.1 +- fix for CVE-2015-3416 + Resolves: #1244727`, + }, + /* + // 7 + { + in: in{ + pack: models.Package{ + Version: "2:7.4.160", + Release: "1.el7", + }, + changelog: `* Mon Dec 12 21:00:00 2016 Karsten Hopp 7.4.160-1.1 + - add fix for CVE-2016-1248 + + * Wed Jan 29 21:00:00 2014 Karsten Hopp 7.4.160-1 + - patchlevel 160 + - Resolves: rhbz#1059321`, + }, + out: `* Mon Dec 12 21:00:00 2016 Karsten Hopp 7.4.160-1.1 + - add fix for CVE-2016-1248`, + }, + // 8 + { + in: in{ + pack: models.Package{ + Version: "2:1.26", + Release: "29.el7", + }, + changelog: `* Mon Jun 20 21:00:00 2016 Pavel Raiskup - 1.26-31 + - avoid double free in selinux code (rhbz#1347396) + + * Thu Jun 4 21:00:00 2015 Pavel Raiskup - 1.26-30 + - don't mistakenly set default ACLs (#1220890) + + * Fri Jan 24 21:00:00 2014 Daniel Mach - 2:1.26-29 + - Mass rebuild 2014-01-24`, + }, + out: `* Mon Jun 20 21:00:00 2016 Pavel Raiskup - 1.26-31 + - avoid double free in selinux code (rhbz#1347396) + + * Thu Jun 4 21:00:00 2015 Pavel Raiskup - 1.26-30 + - don't mistakenly set default ACLs (#1220890)`, + }, + // 9 + { + in: in{ + pack: models.Package{ + Version: "1:1.0.1e", + Release: "51.el7_2.5", + }, + changelog: `* Mon Feb 6 21:00:00 2017 Tomáš Mráz 1.0.1e-60.1 + - fix CVE-2017-3731 - DoS via truncated packets with RC4-MD5 cipher + - fix CVE-2016-8610 - DoS of single-threaded servers via excessive alerts + + * Fri Dec 4 21:00:00 2015 Tomáš Mráz 1.0.1e-52 + - fix CVE-2015-3194 - certificate verify crash with missing PSS parameter + - fix CVE-2015-3195 - X509_ATTRIBUTE memory leak + - fix CVE-2015-3196 - race condition when handling PSK identity hint + + * Tue Jun 23 21:00:00 2015 Tomáš Mráz 1.0.1e-51 + - fix the CVE-2015-1791 fix (broken server side renegotiation)`, + }, + out: `* Mon Feb 6 21:00:00 2017 Tomáš Mráz 1.0.1e-60.1 + - fix CVE-2017-3731 - DoS via truncated packets with RC4-MD5 cipher + - fix CVE-2016-8610 - DoS of single-threaded servers via excessive alerts + + * Fri Dec 4 21:00:00 2015 Tomáš Mráz 1.0.1e-52 + - fix CVE-2015-3194 - certificate verify crash with missing PSS parameter + - fix CVE-2015-3195 - X509_ATTRIBUTE memory leak + - fix CVE-2015-3196 - race condition when handling PSK identity hint`, + }, + // 10 + { + in: in{ + pack: models.Package{ + Version: "1:5.5.47", + Release: "1.el7_2", + }, + changelog: `* Wed Sep 21 21:00:00 2016 Honza Horak - 5.5.52-1 + - Rebase to 5.5.52, that also include fix for CVE-2016-6662 + Resolves: #1377974 + + * Thu Feb 18 21:00:00 2016 Jakub Dorňák - 1:5.5.47-2 + - Add warning to /usr/lib/tmpfiles.d/mariadb.conf + Resolves: #1241623 + + * Wed Feb 3 21:00:00 2016 Jakub Dorňák - 1:5.5.47-1 + - Rebase to 5.5.47 + Also fixes: CVE-2015-4792 CVE-2015-4802 CVE-2015-4815 CVE-2015-4816 + CVE-2015-4819 CVE-2015-4826 CVE-2015-4830 CVE-2015-4836 CVE-2015-4858 + CVE-2015-4861 CVE-2015-4870 CVE-2015-4879 CVE-2015-4913 CVE-2015-7744 + CVE-2016-0505 CVE-2016-0546 CVE-2016-0596 CVE-2016-0597 CVE-2016-0598 + CVE-2016-0600 CVE-2016-0606 CVE-2016-0608 CVE-2016-0609 CVE-2016-0616 + CVE-2016-2047 + Resolves: #1300621`, + }, + out: `* Wed Sep 21 21:00:00 2016 Honza Horak - 5.5.52-1 + - Rebase to 5.5.52, that also include fix for CVE-2016-6662 + Resolves: #1377974 + + * Thu Feb 18 21:00:00 2016 Jakub Dorňák - 1:5.5.47-2 + - Add warning to /usr/lib/tmpfiles.d/mariadb.conf + Resolves: #1241623`, + }, + */ + // 11 + { + in: in{ + pack: models.Package{ + Version: "0.252", + Release: "8.1.el7", + }, + changelog: `* Thu Sep 29 21:00:00 2016 Vitezslav Crhonek - 0.252-8.4 +- Remove wrong entry from usb ids. + Resolves: #1380159 + +* Mon Sep 26 21:00:00 2016 Vitezslav Crhonek - 0.252-8.3 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 + +* Tue Jun 28 21:00:00 2016 Michal Minar 0.252-8.2 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 +- Resolves: rhbz#1291614 +- Resolves: rhbz#1324198 + +* Fri Oct 23 21:00:00 2015 Michal Minar 0.252-8.1 +- Updated pci, usb and vendor ids.`, + }, + out: `* Thu Sep 29 21:00:00 2016 Vitezslav Crhonek - 0.252-8.4 +- Remove wrong entry from usb ids. + Resolves: #1380159 + +* Mon Sep 26 21:00:00 2016 Vitezslav Crhonek - 0.252-8.3 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 + +* Tue Jun 28 21:00:00 2016 Michal Minar 0.252-8.2 +- Updated pci, usb and vendor ids. +- Resolves: rhbz#1292382 +- Resolves: rhbz#1291614 +- Resolves: rhbz#1324198`, + }, + // 12 + { + in: in{ + pack: models.Package{ + Version: "1:2.02", + Release: "0.34.el7_2", + }, + changelog: `* Mon Aug 29 21:00:00 2016 Peter Jones - 2.02-0.44 +- Work around tftp servers that don't work with multiple consecutive slashes in + file paths. + Resolves: rhbz#1217243`, + }, + out: `* Mon Aug 29 21:00:00 2016 Peter Jones - 2.02-0.44 +- Work around tftp servers that don't work with multiple consecutive slashes in + file paths. + Resolves: rhbz#1217243`, }, } - r.Distro = config.Distro{ - Family: "centos", - Release: "5.6", - } - for _, tt := range testsCentos5 { - rpm2changelog, err := r.divideChangelogByPackage(stdoutCentos5) - if err != nil { - t.Errorf("err: %s", err) - } - changelog := r.getChangelogCVELines(rpm2changelog, tt.in) - if tt.out != changelog { - t.Errorf("line: expected %s, actual %s, tt: %#v", tt.out, changelog, tt) + for i, tt := range tests { + diff, _ := r.getDiffChangelog(tt.in.pack, tt.in.changelog) + if tt.out != diff { + t.Errorf("[%d] name: expected \n%s\nactual \n%s", i, tt.out, diff) } } + +} + +func TestDivideChangelogsIntoEachPackages(t *testing.T) { + r := newRedhat(config.ServerInfo{}) + type in struct { + pack models.Package + changelog string + } + + var tests = []struct { + in string + out map[string]string + }{ + { + in: `==================== Available Packages ==================== +1:NetworkManager-1.4.0-20.el7_3.x86_64 rhui-rhel-7-server-rhui-rpms +* Mon Apr 24 21:00:00 2017 Beniamino Galvani - 1:1.4.0-20 +- vlan: use parent interface mtu as default (rh#1414186) + +* Wed Mar 29 21:00:00 2017 Beniamino Galvani - 1:1.4.0-19 +- core: alyways force a sync of the default route (rh#1431268) + + +1:NetworkManager-0.9.9.1-25.git20140326. rhui-rhel-7-server-rhui-optional-rpms +* Tue Jul 1 21:00:00 2014 Jiří Klimeš - 1:0.9.9.1-25.git20140326 +- core: fix MTU handling while merging/subtracting IP configs (rh #1093231) + +* Mon Jun 23 21:00:00 2014 Thomas Haller - 1:0.9.9.1-24.git20140326 +- core: fix crash on failure of reading bridge sysctl values (rh #1112020)`, + out: map[string]string{ + "1:NetworkManager-1.4.0-20.el7_3.x86_64": `* Mon Apr 24 21:00:00 2017 Beniamino Galvani - 1:1.4.0-20 +- vlan: use parent interface mtu as default (rh#1414186) + +* Wed Mar 29 21:00:00 2017 Beniamino Galvani - 1:1.4.0-19 +- core: alyways force a sync of the default route (rh#1431268)`, + + "1:NetworkManager-0.9.9.1-25.git20140326.": `* Tue Jul 1 21:00:00 2014 Jiří Klimeš - 1:0.9.9.1-25.git20140326 +- core: fix MTU handling while merging/subtracting IP configs (rh #1093231) + +* Mon Jun 23 21:00:00 2014 Thomas Haller - 1:0.9.9.1-24.git20140326 +- core: fix crash on failure of reading bridge sysctl values (rh #1112020)`, + }, + }, + } + + for _, tt := range tests { + changelogs := r.divideChangelogsIntoEachPackages(tt.in) + for k, v := range tt.out { + if strings.TrimSpace(v) != strings.TrimSpace(changelogs[k]) { + t.Errorf("expected: %v\nactual: %v", pp.Sprint(tt.out), pp.Sprint(changelogs)) + } + } + } + } diff --git a/scan/serverapi.go b/scan/serverapi.go index a20d1748..86c06cb5 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -59,18 +59,13 @@ type osTypeInterface interface { // osPackages is included by base struct type osPackages struct { // installed packages - Packages models.PackageInfoList + Packages models.Packages // unsecure packages VulnInfos models.VulnInfos -} -func (p *osPackages) setPackages(pi models.PackageInfoList) { - p.Packages = pi -} - -func (p *osPackages) setVulnInfos(vi []models.VulnInfo) { - p.VulnInfos = vi + // kernel information + Kernel models.Kernel } func detectOS(c config.ServerInfo) (osType osTypeInterface) { @@ -421,9 +416,15 @@ func setupChangelogCache() error { needToSetupCache := false for _, s := range servers { switch s.getDistro().Family { - case "ubuntu", "debian", "raspbian": + case config.Raspbian: needToSetupCache = true break + case config.Ubuntu, config.Debian: + //TODO changelopg cache for RedHat, Oracle, Amazon, CentOS is not implemented yet. + if config.Conf.Deep { + needToSetupCache = true + } + break } } if needToSetupCache { @@ -443,6 +444,7 @@ func scanVulns(jsonDir string, scannedAt time.Time, timeoutSec int) error { for _, s := range append(servers, errServers...) { r := s.convertToModel() r.ScannedAt = scannedAt + r.Config.Scan = config.Conf results = append(results, r) } diff --git a/scan/unknownDistro.go b/scan/unknownDistro.go index 30b0ab76..cceaef35 100644 --- a/scan/unknownDistro.go +++ b/scan/unknownDistro.go @@ -26,7 +26,7 @@ func (o *unknown) checkIfSudoNoPasswd() error { return nil } -func (o unknown) checkDependencies() error { +func (o *unknown) checkDependencies() error { return nil } diff --git a/setup/docker/README.md b/setup/docker/README.md index 06cd4e38..0d3c5ac6 100644 --- a/setup/docker/README.md +++ b/setup/docker/README.md @@ -6,6 +6,8 @@ This is the Git repo of the official Docker image for vuls. - go-cve-dictionary - [`latest` (*go-cve-dictionary:latest Dockerfile*)]() +- goval-dictionary + - [`latest` (*goval-dictionary:latest Dockerfile*)]() - vuls - [`latest` (*vuls:latest Dockerfile*)]() - vulsrepo @@ -28,6 +30,14 @@ $ docker run --rm vuls/go-cve-dictionary -v go-cve-dictionary v0.0.xxx xxxx ``` +- goval-dictionary + +```console +$ docker run --rm vuls/goval-dictionary -v + +goval-dictionary v0.0.xxx xxxx +``` + - vuls ```console @@ -44,6 +54,12 @@ vuls v0.0.xxx xxxx $ docker rmi vuls/go-cve-dictionary ``` +- goval-dictionary + +``` +$ docker rmi vuls/goval-dictionary +``` + - vuls ``` @@ -58,6 +74,12 @@ $ docker rmi vuls/vuls $ docker pull vuls/go-cve-dictionary ``` +- goval-dictionary + +``` +$ docker pull vuls/goval-dictionary +``` + - vuls ``` @@ -72,6 +94,12 @@ $ docker run --rm vuls/go-cve-dictionary -v go-cve-dictionary v0.1.xxx xxxx ``` +```console +$ docker run --rm vuls/goval-dictionary -v + +goval-dictionary v0.1.xxx xxxx +``` + - vuls ```console @@ -84,6 +112,7 @@ vuls v0.1.xxx xxxx # How to use this image 1. fetch nvd (vuls/go-cve-dictionary) +1. fetch oval (vuls/goval-dictionary) 1. configuration (vuls/vuls) 1. configtest (vuls/vuls) 1. scan (vuls/vuls) @@ -100,6 +129,19 @@ $ for i in `seq 2002 $(date +"%Y")`; do \ done ``` +- To fetch JVN(Japanese), See [README](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-jvn-data) + +## Step2. Fetch OVAL (e.g. redhat) + +```console +$ docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/goval-dictionary-log:/var/log/vuls \ + vuls/goval-dictionary fetch-redhat 5 6 7 +``` + +- To fetch other OVAL, See [README](https://github.com/kotakanbe/goval-dictionary#usage-fetch-oval-data-from-redhat) + ## Step2. Configuration Create config.toml referring to [this](https://github.com/future-architect/vuls#configuration). @@ -149,6 +191,7 @@ $ docker run --rm -it \ -v /etc/localtime:/etc/localtime:ro \ vuls/vuls report \ -cvedb-path=/vuls/cve.sqlite3 \ + -ovaldb-path=/vuls/oval.sqlite3 \ -format-short-text \ -config=./config.toml # path to config.toml in docker ``` diff --git a/setup/docker/goval-dictionary/latest/Dockerfile b/setup/docker/goval-dictionary/latest/Dockerfile new file mode 100644 index 00000000..6aeeb76b --- /dev/null +++ b/setup/docker/goval-dictionary/latest/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:latest + +MAINTAINER sadayuki-matsuno + +ENV REPOSITORY github.com/kotakanbe/goval-dictionary +ENV LOGDIR /var/log/vuls +ENV WORKDIR /vuls +# goval-dictionary install +RUN git clone https://$REPOSITORY.git $GOPATH/src/$REPOSITORY \ + && cd $GOPATH/src/$REPOSITORY \ + && make install \ + && mkdir -p $LOGDIR + +VOLUME [$WORKDIR, $LOGDIR] +WORKDIR $WORKDIR +ENV PWD $WORKDIR + +ENTRYPOINT ["goval-dictionary"] +CMD ["--help"] diff --git a/setup/docker/goval-dictionary/latest/README.md b/setup/docker/goval-dictionary/latest/README.md new file mode 100644 index 00000000..48b33e68 --- /dev/null +++ b/setup/docker/goval-dictionary/latest/README.md @@ -0,0 +1,125 @@ +# goval-dictionary-Docker + +This is the Git repo of the official Docker image for goval-dictionary. +See the [Hub page](https://hub.docker.com/r/vuls/goval-dictionary/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`latest` (*goval-dictionary:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/goval-dictionary/latest/Dockerfile) + +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. + +- Remove old docker image + +``` +$ docker rmi vuls/goval-dictionary +``` + +- Pull new docker image + +``` +$ docker pull vuls/goval-dictionary +``` + +# What is goval-dictionary? + +This is tool to build a local copy of the OVAL. The local copy is generated in sqlite format, and the tool has a server mode for easy querying. + +# How to use this image + +## check vuls version + +``` +$ docker run --rm vuls/goval-dictionary -v +``` + +## fetch-redhat + +```console +$ for i in `seq 5 7`; do \ + docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/goval-dictionary-log:/var/log/vuls \ + vuls/goval-dictionary fetch-redhat $i; \ + done +``` + +## fetch-debian + +```console +$ for i in `seq 7 10`; do \ + docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/goval-dictionary-log:/var/log/vuls \ + vuls/goval-dictionary fetch-debian $i; \ + done +``` + +## fetch-ubuntu + +```console +$ for i in `seq 12 2 16`; do \ + docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/goval-dictionary-log:/var/log/vuls \ + vuls/goval-dictionary fetch-ubuntu $i; \ + done +``` + +## fetch-suse + +```console +$ docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/goval-dictionary-log:/var/log/vuls \ + vuls/goval-dictionary fetch-suse -opensuse 13.2 +``` + +## fetch-oracle + +```console +$ docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/goval-dictionary-log:/var/log/vuls \ + vuls/goval-dictionary fetch-oracle +``` + +## server + +```console +$ docker run -dt \ + --name goval-dictionary \ + -v $PWD:/vuls \ + -v $PWD/goval-dictionary-log:/var/log/vuls \ + --expose 1324 \ + -p 1324:1324 \ + vuls/goval-dictionary server --bind=0.0.0.0 +``` + +Prease refer to [this](https://hub.docker.com/r/vuls/goval-dictionary). + +## vuls + +Please refer to [this](https://hub.docker.com/r/vuls/vuls/). + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory](https://github.com/future-architect/vuls/tree/master/setup/docker) of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues + +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). + +## Contributing + +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/util/logutil.go b/util/logutil.go index 021462ea..653fdf85 100644 --- a/util/logutil.go +++ b/util/logutil.go @@ -22,8 +22,8 @@ import ( "path/filepath" "runtime" - "github.com/Sirupsen/logrus" "github.com/rifflock/lfshook" + "github.com/sirupsen/logrus" "github.com/future-architect/vuls/config" formatter "github.com/kotakanbe/logrus-prefixed-formatter" diff --git a/util/util.go b/util/util.go index 3b94e8a6..da888b2c 100644 --- a/util/util.go +++ b/util/util.go @@ -135,3 +135,15 @@ func Truncate(str string, length int) string { } return str } + +// Distinct a slice +func Distinct(ss []string) (distincted []string) { + m := map[string]bool{} + for _, s := range ss { + if _, found := m[s]; !found { + m[s] = true + distincted = append(distincted, s) + } + } + return +}