Compare commits

...

4 Commits

Author SHA1 Message Date
Kota Kanbe
169db6e320 sort 2021-11-08 12:29:15 +09:00
Kota Kanbe
79c7fc972c reverse deps recursively 2021-11-05 10:01:18 +09:00
Kota Kanbe
fb9cebd3e2 feat reversedeps 2021-11-04 10:47:19 +09:00
Kota Kanbe
43d987e200 feat(scan) detect the dependencies for each updatable packages 2021-11-01 12:45:09 +09:00
6 changed files with 474 additions and 30 deletions

5
go.mod
View File

@@ -16,7 +16,7 @@ require (
github.com/briandowns/spinner v1.16.0 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheggaaa/pb/v3 v3.0.8 // indirect
github.com/cheggaaa/pb v1.0.27
github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b
github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21
github.com/emersion/go-smtp v0.14.0
@@ -87,6 +87,7 @@ require (
github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect
github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect
github.com/caarlos0/env/v6 v6.0.0 // indirect
github.com/cheggaaa/pb/v3 v3.0.8 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
@@ -142,7 +143,7 @@ require (
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect

4
go.sum
View File

@@ -2013,8 +2013,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=

View File

@@ -72,6 +72,31 @@ func (ps Packages) FindByFQPN(nameVerRel string) (*Package, error) {
return nil, xerrors.Errorf("Failed to find the package: %s", nameVerRel)
}
// ResolveReverseDepsRecursively resolve and set reverse dependencies for each packages recursively
func (ps Packages) ResolveReverseDepsRecursively() {
for name, pkg := range ps {
depsmap := ps.resolveReverseDeps(pkg, map[string]struct{}{})
delete(depsmap, pkg.Name)
for depname := range depsmap {
pkg.ReverseDependenciesRecursively = append(pkg.ReverseDependenciesRecursively, depname)
}
ps[name] = pkg
}
}
func (ps Packages) resolveReverseDeps(pkg Package, acc map[string]struct{}) map[string]struct{} {
acc[pkg.Name] = struct{}{}
for _, name := range pkg.ReverseDependencies {
if _, ok := acc[name]; ok {
continue
}
if p, ok := ps[name]; ok {
acc = ps.resolveReverseDeps(p, acc)
}
}
return acc
}
// Package has installed binary packages.
type Package struct {
Name string `json:"name"`
@@ -84,6 +109,18 @@ type Package struct {
Changelog *Changelog `json:"changelog,omitempty"`
AffectedProcs []AffectedProcess `json:",omitempty"`
NeedRestartProcs []NeedRestartProcess `json:",omitempty"`
// Dependencies are dependent packages
Dependencies []string `json:",omitempty"`
// ReverseDependencies are packages which depend on this package
ReverseDependencies []string `json:",omitempty"`
// ReverseDependencies are packages which depend on this package
ReverseDependenciesRecursively []string `json:",omitempty"`
// DependenciesForUpdate are packages that needs to be updated together
DependenciesForUpdate []string `json:",omitempty"`
}
// FQPN returns Fully-Qualified-Package-Name

View File

@@ -428,3 +428,166 @@ func Test_NewPortStat(t *testing.T) {
})
}
}
func TestPackages_resolveReverseDeps(t *testing.T) {
type args struct {
pkg Package
acc map[string]struct{}
}
tests := []struct {
name string
ps Packages
args args
want map[string]struct{}
}{
{
name: "",
ps: map[string]Package{
"pkgA": {
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
},
"pkgB": {
Name: "pkgB",
ReverseDependencies: []string{"pkgC"},
},
"pkgC": {
Name: "pkgC",
ReverseDependencies: []string{""},
},
},
args: args{
pkg: Package{
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
},
acc: map[string]struct{}{},
},
want: map[string]struct{}{
"pkgA": {},
"pkgB": {},
"pkgC": {},
},
},
{
name: "",
ps: map[string]Package{
"pkgA": {
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
},
"pkgB": {
Name: "pkgB",
ReverseDependencies: []string{"pkgA", "pkgC"},
},
"pkgC": {
Name: "pkgC",
ReverseDependencies: []string{"pkgB"},
},
},
args: args{
pkg: Package{
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
},
acc: map[string]struct{}{},
},
want: map[string]struct{}{
"pkgA": {},
"pkgB": {},
"pkgC": {},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.ps.resolveReverseDeps(tt.args.pkg, tt.args.acc); !reflect.DeepEqual(got, tt.want) {
t.Errorf("Packages.resolveReverseDeps() = %v, want %v", got, tt.want)
}
})
}
}
func TestPackages_resolveReverseDepsRecursively(t *testing.T) {
tests := []struct {
name string
ps Packages
want Packages
}{
{
name: "",
ps: map[string]Package{
"pkgA": {
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
},
"pkgB": {
Name: "pkgB",
ReverseDependencies: []string{"pkgC"},
},
"pkgC": {
Name: "pkgC",
ReverseDependencies: []string{""},
},
},
want: map[string]Package{
"pkgA": {
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
ReverseDependenciesRecursively: []string{"pkgB", "pkgC"},
},
"pkgB": {
Name: "pkgB",
ReverseDependencies: []string{"pkgC"},
ReverseDependenciesRecursively: []string{"pkgC"},
},
"pkgC": {
Name: "pkgC",
ReverseDependencies: []string{""},
},
},
},
{
name: "",
ps: map[string]Package{
"pkgA": {
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
},
"pkgB": {
Name: "pkgB",
ReverseDependencies: []string{"pkgA", "pkgC"},
},
"pkgC": {
Name: "pkgC",
ReverseDependencies: []string{"pkgB"},
},
},
want: map[string]Package{
"pkgA": {
Name: "pkgA",
ReverseDependencies: []string{"pkgB"},
ReverseDependenciesRecursively: []string{"pkgB", "pkgC"},
},
"pkgB": {
Name: "pkgB",
ReverseDependencies: []string{"pkgA", "pkgC"},
ReverseDependenciesRecursively: []string{"pkgA", "pkgC"},
},
"pkgC": {
Name: "pkgC",
ReverseDependencies: []string{"pkgB"},
ReverseDependenciesRecursively: []string{"pkgA", "pkgB"},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.ps.ResolveReverseDepsRecursively()
if !reflect.DeepEqual(tt.ps, tt.want) {
t.Errorf("Packages.resolveReverseDepsRecursively() = \n%+v, want \n%+v", tt.ps, tt.want)
}
})
}
}

View File

@@ -4,8 +4,10 @@ import (
"bufio"
"fmt"
"regexp"
"sort"
"strings"
"github.com/cheggaaa/pb"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/constant"
"github.com/future-architect/vuls/logging"
@@ -218,6 +220,10 @@ func (o *redhatBase) execCheckDeps(packNames []string) error {
return nil
}
func (o *redhatBase) isDnf() bool {
return o.exec(util.PrependProxyEnv(`repoquery --version | grep dnf`), o.sudo.repoquery()).isSuccess()
}
func (o *redhatBase) preCure() error {
if err := o.detectIPAddr(); err != nil {
o.log.Warnf("Failed to detect IP addresses: %s", err)
@@ -230,8 +236,7 @@ func (o *redhatBase) preCure() error {
func (o *redhatBase) postScan() error {
if o.isExecYumPS() {
if err := o.pkgPs(o.getOwnerPkgs); err != nil {
err = xerrors.Errorf("Failed to execute yum-ps: %w", err)
o.log.Warnf("err: %+v", err)
o.log.Warnf("err: %+v", xerrors.Errorf("Failed to execute yum-ps: %w", err))
o.warns = append(o.warns, err)
// Only warning this error
}
@@ -239,8 +244,7 @@ func (o *redhatBase) postScan() error {
if o.isExecNeedsRestarting() {
if err := o.needsRestarting(); err != nil {
err = xerrors.Errorf("Failed to execute need-restarting: %w", err)
o.log.Warnf("err: %+v", err)
o.log.Warnf("err: %+v", xerrors.Errorf("Failed to execute need-restarting: %w", err))
o.warns = append(o.warns, err)
// Only warning this error
}
@@ -260,15 +264,10 @@ func (o *redhatBase) scanPackages() (err error) {
return xerrors.Errorf("Failed to scan installed packages: %w", err)
}
if o.EnabledDnfModules, err = o.detectEnabledDnfModules(); err != nil {
return xerrors.Errorf("Failed to detect installed dnf modules: %w", err)
}
fn := func(pkgName string) execResult { return o.exec(fmt.Sprintf("rpm -q --last %s", pkgName), noSudo) }
o.Kernel.RebootRequired, err = o.rebootRequired(fn)
if err != nil {
err = xerrors.Errorf("Failed to detect the kernel reboot required: %w", err)
o.log.Warnf("err: %+v", err)
o.log.Warnf("err: %+v", xerrors.Errorf("Failed to detect the kernel reboot required: %w", err))
o.warns = append(o.warns, err)
// Only warning this error
}
@@ -281,15 +280,31 @@ func (o *redhatBase) scanPackages() (err error) {
}
}
if err := o.yumMakeCache(); err != nil {
return xerrors.Errorf("Failed to `yum makecache`: %w", err)
}
// `dnf module list` needs yum cache
if o.EnabledDnfModules, err = o.detectEnabledDnfModules(); err != nil {
return xerrors.Errorf("Failed to detect installed dnf modules: %w", err)
}
updatable, err := o.scanUpdatablePackages()
if err != nil {
err = xerrors.Errorf("Failed to scan updatable packages: %w", err)
o.log.Warnf("err: %+v", err)
o.log.Warnf("err: %+v", xerrors.Errorf("Failed to scan updatable packages: %w", err))
o.warns = append(o.warns, err)
// Only warning this error
} else {
o.Packages.MergeNewVersion(updatable)
}
if o.getServerInfo().Mode.IsDeep() {
resolver := yumDependentResolver{redhat: o, isDnf: o.isDnf()}
resolver.detectDependenciesForUpdate()
resolver.detectReverseDependencies()
o.Packages.ResolveReverseDepsRecursively()
}
return nil
}
@@ -420,13 +435,8 @@ func (o *redhatBase) yumMakeCache() error {
}
func (o *redhatBase) scanUpdatablePackages() (models.Packages, error) {
if err := o.yumMakeCache(); err != nil {
return nil, xerrors.Errorf("Failed to `yum makecache`: %w", err)
}
isDnf := o.exec(util.PrependProxyEnv(`repoquery --version | grep dnf`), o.sudo.repoquery()).isSuccess()
cmd := `repoquery --all --pkgnarrow=updates --qf='%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{REPO}'`
if isDnf {
if o.isDnf() {
cmd = `repoquery --upgrades --qf='%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{REPONAME}' -q`
}
for _, repo := range o.getServerInfo().Enablerepo {
@@ -700,9 +710,6 @@ func (o *redhatBase) detectEnabledDnfModules() ([]string, error) {
cmd := `dnf --nogpgcheck --cacheonly --color=never --quiet module list --enabled`
r := o.exec(util.PrependProxyEnv(cmd), noSudo)
if !r.isSuccess() {
if strings.Contains(r.Stdout, "Cache-only enabled but no cache") {
return nil, xerrors.Errorf("sudo yum check-update to make local cache before scanning: %s", r)
}
return nil, xerrors.Errorf("Failed to dnf module list: %s", r)
}
return o.parseDnfModuleList(r.Stdout)
@@ -723,3 +730,122 @@ func (o *redhatBase) parseDnfModuleList(stdout string) (labels []string, err err
}
return
}
type yumDependentResolver struct {
redhat *redhatBase
isDnf bool
}
func (o *yumDependentResolver) detectDependenciesForUpdate() {
o.redhat.log.Infof("Detecting the dependencies for each packages when yum updating")
o.redhat.log.Infof("If it is too slow, `yum clean all` may make it faster")
bar := pb.StartNew(len(o.redhat.Packages))
for name, pkg := range o.redhat.Packages {
bar.Increment()
if pkg.Version == pkg.NewVersion && pkg.Release == pkg.NewRelease {
// only updatable pkgs
continue
}
names, err := o.scanUpdatablePkgDeps(name)
if err != nil {
o.redhat.log.Warnf("err: %+v", xerrors.Errorf("Failed to scan dependent packages for update: %w", err))
o.redhat.warns = append(o.redhat.warns, err)
// Only warning this error
}
sort.Strings(names)
pkg.DependenciesForUpdate = names
o.redhat.Packages[name] = pkg
}
bar.Finish()
return
}
func (o *yumDependentResolver) scanUpdatablePkgDeps(name string) (depsPkgNames []string, err error) {
cmd := fmt.Sprintf("LANGUAGE=en_US.UTF-8 yum install --assumeno --cacheonly %s", name)
r := o.redhat.exec(cmd, true)
if !r.isSuccess(0, 1) {
return nil, xerrors.Errorf("Failed to SSH: %s", r)
}
names := o.parseYumInstall(r.Stdout)
for _, n := range names {
if _, ok := o.redhat.Packages[n]; ok {
depsPkgNames = append(depsPkgNames, n)
}
}
return
}
func (o *yumDependentResolver) parseYumInstall(stdout string) []string {
names, inDepsLines := []string{}, false
scanner := bufio.NewScanner(strings.NewReader(stdout))
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Updating for dependencies:") {
inDepsLines = true
continue
}
if inDepsLines {
ss := strings.Fields(line)
if len(ss) == 0 {
break
}
names = append(names, ss[0])
}
}
return names
}
func (o *yumDependentResolver) detectReverseDependencies() {
o.redhat.log.Infof("Detecting the reverse dependencies for each packages")
bar := pb.StartNew(len(o.redhat.Packages))
for name, pkg := range o.redhat.Packages {
bar.Increment()
// if pkg.Version == pkg.NewVersion && pkg.Release == pkg.NewRelease {
// continue
// }
names, err := o.repoqueryWhatRequires(name)
if err != nil {
o.redhat.log.Warnf("err: %+v", xerrors.Errorf("Failed to scan reverse dependent packages: %w", err))
o.redhat.warns = append(o.redhat.warns, err)
// Only warning this error
}
sort.Strings(names)
pkg.ReverseDependencies = names
o.redhat.Packages[name] = pkg
}
bar.Finish()
return
}
func (o *yumDependentResolver) repoqueryWhatRequires(pkgName string) (depsPkgNames []string, err error) {
cmd := `LANGUAGE=en_US.UTF-8 repoquery --cache --resolve --pkgnarrow=installed --qf "%{name}" --whatrequires ` + pkgName
if o.isDnf {
cmd = `LANGUAGE=en_US.UTF-8 repoquery --cache --installed --qf "%{name}" --whatrequires ` + pkgName
}
r := o.redhat.exec(cmd, true)
if !r.isSuccess() {
return nil, xerrors.Errorf("Failed to SSH: %s", r)
}
names := o.parseRepoqueryWhatRequires(r.Stdout)
for _, n := range names {
if pkgName == n {
continue
}
if _, ok := o.redhat.Packages[n]; ok {
depsPkgNames = append(depsPkgNames, n)
}
}
return
}
func (o *yumDependentResolver) parseRepoqueryWhatRequires(stdout string) []string {
names := []string{}
scanner := bufio.NewScanner(strings.NewReader(stdout))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
names = append(names, line)
}
}
return names
}

View File

@@ -10,11 +10,6 @@ import (
"github.com/k0kubun/pp"
)
// func unixtimeNoerr(s string) time.Time {
// t, _ := unixtime(s)
// return t
// }
func TestParseInstalledPackagesLinesRedhat(t *testing.T) {
r := newRHEL(config.ServerInfo{})
r.Distro = config.Distro{Family: constant.RedHat}
@@ -641,3 +636,125 @@ kernel-3.10.0-1062.12.1.el7.x86_64 Sat 29 Feb 2020 12:09:00 PM UTC`,
})
}
}
func Test_redhatBase_parseYumInstall(t *testing.T) {
type fields struct {
base base
sudo rootPriv
}
type args struct {
stdout string
}
tests := []struct {
name string
fields fields
args args
wantPkgNames []string
}{
{
name: "ok",
fields: fields{
base: base{},
},
args: args{
stdout: `===========================================================================================================================================================
Package Arch Version Repository Size
===========================================================================================================================================================
Updating:
glibc x86_64 2.17-325.el7_9 updates 3.6 M
Updating for dependencies:
glibc-common x86_64 2.17-325.el7_9 updates 12 M
glibc-devel x86_64 2.17-325.el7_9 updates 1.1 M
glibc-headers x86_64 2.17-325.el7_9 updates 691 k
Transaction Summary
===========================================================================================================================================================
Upgrade 1 Package (+3 Dependent packages)
Exiting on user command
Your transaction was saved, rerun it with:
yum load-transaction /tmp/yum_save_tx.2021-10-25.08-24.1EXcRc.yumtx`,
},
wantPkgNames: []string{
"glibc-common",
"glibc-devel",
"glibc-headers",
},
},
{
name: "no deps",
fields: fields{
base: base{},
},
args: args{
stdout: `Package zlib-1.2.7-19.el7_9.x86_64 already installed and latest version`,
},
wantPkgNames: []string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
o := &redhatBase{
base: tt.fields.base,
}
resolver := yumDependentResolver{redhat: o}
if gotPkgNames := resolver.parseYumInstall(tt.args.stdout); !reflect.DeepEqual(gotPkgNames, tt.wantPkgNames) {
t.Errorf("redhatBase.parseYumInstall() = %v, want %v", gotPkgNames, tt.wantPkgNames)
}
})
}
}
func Test_yumDependentResolver_parseRepoqueryWhatRequires(t *testing.T) {
type fields struct {
redhat *redhatBase
}
type args struct {
stdout string
}
tests := []struct {
name string
fields fields
args args
want []string
}{
{
name: "no deps",
fields: fields{redhat: &redhatBase{}},
args: args{
stdout: "",
},
want: []string{},
},
{
name: "no deps",
fields: fields{redhat: &redhatBase{}},
args: args{
stdout: `which
whois
wpa_supplicant
xcb-util
xfsprogs
xinetd`,
},
want: []string{
"which",
"whois",
"wpa_supplicant",
"xcb-util",
"xfsprogs",
"xinetd",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
o := &yumDependentResolver{
redhat: tt.fields.redhat,
}
if got := o.parseRepoqueryWhatRequires(tt.args.stdout); !reflect.DeepEqual(got, tt.want) {
t.Errorf("yumDependentResolver.parseRepoqueryWhatRequires() = %v, want %v", got, tt.want)
}
})
}
}