0
0
Fork 0
mirror of https://github.com/crazy-max/diun.git synced 2025-03-17 12:52:40 +00:00

Implement Docker provider ()

This commit is contained in:
CrazyMax 2019-12-13 23:04:02 +01:00
parent cc7e6cd13b
commit 32438dbb2e
No known key found for this signature in database
GPG key ID: 3248E46B6BB8C7F7
17 changed files with 447 additions and 221 deletions

View file

@ -7,7 +7,10 @@ import (
"github.com/crazy-max/diun/internal/config"
"github.com/crazy-max/diun/internal/db"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/notif"
dockerPrd "github.com/crazy-max/diun/internal/provider/docker"
imagePrd "github.com/crazy-max/diun/internal/provider/image"
"github.com/hako/durafmt"
"github.com/panjf2000/ants/v2"
"github.com/robfig/cron/v3"
@ -73,7 +76,7 @@ func (di *Diun) Start() error {
select {}
}
// Run runs diun process
// Run starts diun
func (di *Diun) Run() {
if !atomic.CompareAndSwapUint32(&di.locker, 0, 1) {
log.Warn().Msg("Already running")
@ -89,19 +92,27 @@ func (di *Diun) Run() {
log.Info().Msg("Starting Diun...")
di.wg = new(sync.WaitGroup)
di.pool, _ = ants.NewPoolWithFunc(di.cfg.Watch.Workers, func(i interface{}) {
var err error
switch t := i.(type) {
case imageJob:
err = di.imageJob(t)
if err != nil {
log.Error().Err(err).Msg("Job image error")
}
job := i.(model.Job)
if err := di.runJob(job); err != nil {
log.Error().Err(err).
Str("provider", job.Provider).
Str("id", job.ID).
Msg("Cannot run job")
}
di.wg.Done()
})
defer di.pool.Release()
di.procImages()
// Docker provider
for _, job := range dockerPrd.New(di.cfg.Providers.Docker).ListJob() {
di.createJob(job)
}
// Image provider
for _, job := range imagePrd.New(di.cfg.Providers.Image).ListJob() {
di.createJob(job)
}
di.wg.Wait()
}

View file

@ -1,137 +0,0 @@
package app
import (
"fmt"
"time"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/model/provider"
"github.com/crazy-max/diun/internal/utl"
"github.com/crazy-max/diun/pkg/docker"
"github.com/crazy-max/diun/pkg/docker/registry"
"github.com/rs/zerolog/log"
)
type imageJob struct {
image provider.Image
registry *docker.RegistryClient
}
func (di *Diun) procImages() {
// Iterate images
for _, img := range di.cfg.Providers.Image {
regOpts := di.cfg.RegOpts[img.RegOptsID]
reg, err := docker.NewRegistryClient(docker.RegistryOptions{
Os: img.Os,
Arch: img.Arch,
Username: regOpts.Username,
Password: regOpts.Password,
Timeout: time.Duration(regOpts.Timeout) * time.Second,
InsecureTLS: regOpts.InsecureTLS,
})
if err != nil {
log.Error().Err(err).Str("image", img.Name).Msg("Cannot create registry client")
continue
}
image, err := registry.ParseImage(img.Name)
if err != nil {
log.Error().Err(err).Str("image", img.Name).Msg("Cannot parse image")
continue
}
di.wg.Add(1)
err = di.pool.Invoke(imageJob{
image: img,
registry: reg,
})
if err != nil {
log.Error().Err(err).Msgf("Invoking image job")
}
if !img.WatchRepo || image.Domain == "" {
continue
}
tags, err := reg.Tags(docker.TagsOptions{
Image: image,
Max: img.MaxTags,
Include: img.IncludeTags,
Exclude: img.ExcludeTags,
})
if err != nil {
log.Error().Err(err).Str("image", image.String()).Msg("Cannot retrieve tags")
continue
}
log.Debug().Str("image", image.String()).Msgf("%d tag(s) found in repository. %d will be analyzed (%d max, %d not included, %d excluded).",
tags.Total,
len(tags.List),
img.MaxTags,
tags.NotIncluded,
tags.Excluded,
)
for _, tag := range tags.List {
img.Name = fmt.Sprintf("%s/%s:%s", image.Domain, image.Path, tag)
di.wg.Add(1)
err = di.pool.Invoke(imageJob{
image: img,
registry: reg,
})
if err != nil {
log.Error().Err(err).Msgf("Invoking image job (tag)")
}
}
}
}
func (di *Diun) imageJob(job imageJob) error {
image, err := registry.ParseImage(job.image.Name)
if err != nil {
return err
}
if !utl.IsIncluded(image.Tag, job.image.IncludeTags) {
log.Warn().Str("image", image.String()).Msg("Tag not included")
return nil
} else if utl.IsExcluded(image.Tag, job.image.ExcludeTags) {
log.Warn().Str("image", image.String()).Msg("Tag excluded")
return nil
}
liveManifest, err := job.registry.Manifest(image)
if err != nil {
return err
}
dbManifest, err := di.db.GetManifest(image)
if err != nil {
return err
}
status := model.ImageStatusUnchange
if dbManifest.Name == "" {
status = model.ImageStatusNew
log.Info().Str("image", image.String()).Msg("New image found")
} else if !liveManifest.Created.Equal(*dbManifest.Created) {
status = model.ImageStatusUpdate
log.Info().Str("image", image.String()).Msg("Image update found")
} else {
log.Debug().Str("image", image.String()).Msg("No changes")
return nil
}
if err := di.db.PutManifest(image, liveManifest); err != nil {
return err
}
log.Debug().Str("image", image.String()).Msg("Manifest saved to database")
di.notif.Send(model.NotifEntry{
Status: status,
Image: image,
Manifest: liveManifest,
})
return nil
}

149
internal/app/job.go Normal file
View file

@ -0,0 +1,149 @@
package app
import (
"fmt"
"time"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/utl"
"github.com/crazy-max/diun/pkg/docker"
"github.com/crazy-max/diun/pkg/docker/registry"
"github.com/rs/zerolog/log"
)
func (di *Diun) createJob(job model.Job) {
sublog := log.With().
Str("provider", job.Provider).
Str("id", job.ID).
Str("image", job.Image.Name).
Logger()
regOpts, err := di.getRegOpts(job.Image.RegOptsID)
if err != nil {
sublog.Warn().Err(err).Msg("Registry options")
}
job.Registry, err = docker.NewRegistryClient(docker.RegistryOptions{
Os: job.Image.Os,
Arch: job.Image.Arch,
Username: regOpts.Username,
Password: regOpts.Password,
Timeout: time.Duration(regOpts.Timeout) * time.Second,
InsecureTLS: regOpts.InsecureTLS,
})
if err != nil {
sublog.Error().Err(err).Msg("Cannot create registry client")
return
}
regimg, err := registry.ParseImage(job.Image.Name)
if err != nil {
sublog.Error().Err(err).Msg("Cannot parse image")
return
}
di.wg.Add(1)
err = di.pool.Invoke(job)
if err != nil {
sublog.Error().Err(err).Msgf("Invoking job")
}
if !job.Image.WatchRepo || regimg.Domain == "" {
return
}
tags, err := job.Registry.Tags(docker.TagsOptions{
Image: regimg,
Max: job.Image.MaxTags,
Include: job.Image.IncludeTags,
Exclude: job.Image.ExcludeTags,
})
if err != nil {
sublog.Error().Err(err).Msg("Cannot list tags from registry")
return
}
log.Debug().Str("image", regimg.String()).Msgf("%d tag(s) found in repository. %d will be analyzed (%d max, %d not included, %d excluded).",
tags.Total,
len(tags.List),
job.Image.MaxTags,
tags.NotIncluded,
tags.Excluded,
)
for _, tag := range tags.List {
job.Image.Name = fmt.Sprintf("%s/%s:%s", regimg.Domain, regimg.Path, tag)
di.wg.Add(1)
err = di.pool.Invoke(job)
if err != nil {
sublog.Error().Err(err).Msgf("Invoking job (tag)")
}
}
}
func (di *Diun) runJob(job model.Job) error {
image, err := registry.ParseImage(job.Image.Name)
if err != nil {
return err
}
sublog := log.With().
Str("provider", job.Provider).
Str("id", job.ID).
Str("image", image.String()).
Logger()
if !utl.IsIncluded(image.Tag, job.Image.IncludeTags) {
sublog.Warn().Msg("Tag not included")
return nil
} else if utl.IsExcluded(image.Tag, job.Image.ExcludeTags) {
sublog.Warn().Msg("Tag excluded")
return nil
}
liveManifest, err := job.Registry.Manifest(image)
if err != nil {
return err
}
dbManifest, err := di.db.GetManifest(image)
if err != nil {
return err
}
status := model.ImageStatusUnchange
if dbManifest.Name == "" {
status = model.ImageStatusNew
sublog.Info().Msg("New image found")
} else if !liveManifest.Created.Equal(*dbManifest.Created) {
status = model.ImageStatusUpdate
sublog.Info().Msg("Image update found")
} else {
sublog.Debug().Msg("No changes")
return nil
}
if err := di.db.PutManifest(image, liveManifest); err != nil {
return err
}
sublog.Debug().Msg("Manifest saved to database")
di.notif.Send(model.NotifEntry{
Status: status,
Image: image,
Manifest: liveManifest,
})
return nil
}
func (di *Diun) getRegOpts(id string) (model.RegOpts, error) {
if id == "" {
return model.RegOpts{}, nil
}
if regopts, ok := di.cfg.RegOpts[id]; ok {
return regopts, nil
} else {
return model.RegOpts{}, fmt.Errorf("%s not found", id)
}
}

View file

@ -11,7 +11,6 @@ import (
"regexp"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/model/provider"
"github.com/crazy-max/diun/internal/utl"
"github.com/imdario/mergo"
"github.com/rs/zerolog/log"
@ -64,7 +63,7 @@ func Load(flags model.Flags, version string) (*Config, error) {
},
},
Providers: model.Providers{
Image: []provider.Image{},
Image: []model.PrdImage{},
},
}
@ -142,18 +141,15 @@ func (cfg *Config) validateRegOpts(id string, regopts model.RegOpts) error {
return nil
}
func (cfg *Config) validateDockerProvider(key int, dock provider.Docker) error {
func (cfg *Config) validateDockerProvider(key int, dock model.PrdDocker) error {
if dock.ID == "" {
return fmt.Errorf("ID is required for docker provider %d", key)
}
if err := mergo.Merge(&dock, provider.Docker{
Endpoint: os.Getenv("DOCKER_HOST"),
ApiVersion: os.Getenv("DOCKER_API_VERSION"),
CertPath: os.Getenv("DOCKER_CERT_PATH"),
TLSVerify: os.Getenv("DOCKER_TLS_VERIFY"),
SwarmMode: false,
WatchStopped: false,
if err := mergo.Merge(&dock, model.PrdDocker{
SwarmMode: false,
WatchByDefault: false,
WatchStopped: false,
}); err != nil {
return fmt.Errorf("cannot set default docker provider values for %s: %v", dock.ID, err)
}
@ -162,12 +158,12 @@ func (cfg *Config) validateDockerProvider(key int, dock provider.Docker) error {
return nil
}
func (cfg *Config) validateImageProvider(key int, img provider.Image) error {
func (cfg *Config) validateImageProvider(key int, img model.PrdImage) error {
if img.Name == "" {
return fmt.Errorf("name is required for image provider %d", key)
}
if err := mergo.Merge(&img, provider.Image{
if err := mergo.Merge(&img, model.PrdImage{
Os: "linux",
Arch: "amd64",
WatchRepo: false,
@ -176,13 +172,6 @@ func (cfg *Config) validateImageProvider(key int, img provider.Image) error {
return fmt.Errorf("cannot set default image image values for %s: %v", img.Name, err)
}
if img.RegOptsID != "" {
_, found := cfg.RegOpts[img.RegOptsID]
if !found {
return fmt.Errorf("registry options %s not found for %s", img.RegOptsID, img.Name)
}
}
for _, includeTag := range img.IncludeTags {
if _, err := regexp.Compile(includeTag); err != nil {
return fmt.Errorf("include tag regex '%s' for %s cannot compile, %v", includeTag, img.Name, err)

View file

@ -34,10 +34,8 @@ regopts:
providers:
docker:
- id: swarm
endpoint: unix:///var/run/docker.sock
api_version: 1.13
swarm_mode: true
- id: local
watch_by_default: true
image:
- name: docker.io/crazymax/nextcloud:latest
regopts_id: someregopts

View file

@ -5,7 +5,6 @@ import (
"github.com/crazy-max/diun/internal/config"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/model/provider"
"github.com/stretchr/testify/assert"
)
@ -81,15 +80,13 @@ func TestLoad(t *testing.T) {
},
},
Providers: model.Providers{
Docker: []provider.Docker{
Docker: []model.PrdDocker{
{
ID: "swarm",
Endpoint: "unix:///var/run/docker.sock",
ApiVersion: "1.13",
SwarmMode: true,
ID: "local",
WatchByDefault: true,
},
},
Image: []provider.Image{
Image: []model.PrdImage{
{
Name: "docker.io/crazymax/nextcloud:latest",
Os: "linux",

View file

@ -9,12 +9,3 @@ type App struct {
Author string
Version string
}
const (
ImageStatusNew = ImageStatus("new")
ImageStatusUpdate = ImageStatus("update")
ImageStatusUnchange = ImageStatus("unchange")
)
// ImageStatus holds Docker image status analysis
type ImageStatus string

View file

@ -1,6 +1,6 @@
package provider
package model
// Image holds image provider configuration
// Image holds image configuration
type Image struct {
Name string `yaml:"name,omitempty" json:",omitempty"`
Os string `yaml:"os,omitempty" json:",omitempty"`
@ -11,3 +11,12 @@ type Image struct {
IncludeTags []string `yaml:"include_tags,omitempty" json:",omitempty"`
ExcludeTags []string `yaml:"exclude_tags,omitempty" json:",omitempty"`
}
const (
ImageStatusNew = ImageStatus("new")
ImageStatusUpdate = ImageStatus("update")
ImageStatusUnchange = ImageStatus("unchange")
)
// ImageStatus holds Docker image status analysis
type ImageStatus string

11
internal/model/job.go Normal file
View file

@ -0,0 +1,11 @@
package model
import "github.com/crazy-max/diun/pkg/docker"
// Job holds job configuration
type Job struct {
Provider string
ID string
Image Image
Registry *docker.RegistryClient
}

View file

@ -1,14 +0,0 @@
package provider
// Docker holds docker provider configuration
type Docker struct {
ID string `yaml:"id,omitempty" json:",omitempty"`
Endpoint string `yaml:"endpoint,omitempty" json:",omitempty"`
ApiVersion string `yaml:"api_version,omitempty" json:",omitempty"`
CAFile string `yaml:"ca_file,omitempty" json:",omitempty"`
CertFile string `yaml:"cert_file,omitempty" json:",omitempty"`
KeyFile string `yaml:"key_file,omitempty" json:",omitempty"`
TLSVerify string `yaml:"tls_verify,omitempty" json:",omitempty"`
SwarmMode bool `yaml:"swarm_mode,omitempty" json:",omitempty"`
WatchStopped bool `yaml:"watch_stopped,omitempty" json:",omitempty"`
}

View file

@ -1,9 +1,24 @@
package model
import "github.com/crazy-max/diun/internal/model/provider"
// Providers represents a provider configuration
type Providers struct {
Image []provider.Image `yaml:"image,omitempty" json:",omitempty"`
Docker []provider.Docker `yaml:"docker,omitempty" json:",omitempty"`
Image []PrdImage `yaml:"image,omitempty" json:",omitempty"`
Docker []PrdDocker `yaml:"docker,omitempty" json:",omitempty"`
}
// PrdImage holds image provider configuration
type PrdImage Image
// PrdDocker holds docker provider configuration
type PrdDocker struct {
ID string `yaml:"id,omitempty" json:",omitempty"`
Endpoint string `yaml:"endpoint,omitempty" json:",omitempty"`
ApiVersion string `yaml:"api_version,omitempty" json:",omitempty"`
CAFile string `yaml:"ca_file,omitempty" json:",omitempty"`
CertFile string `yaml:"cert_file,omitempty" json:",omitempty"`
KeyFile string `yaml:"key_file,omitempty" json:",omitempty"`
TLSVerify string `yaml:"tls_verify,omitempty" json:",omitempty"`
SwarmMode bool `yaml:"swarm_mode,omitempty" json:",omitempty"`
WatchByDefault bool `yaml:"watch_by_default,omitempty" json:",omitempty"`
WatchStopped bool `yaml:"watch_stopped,omitempty" json:",omitempty"`
}

View file

@ -0,0 +1,98 @@
package docker
import (
"fmt"
"reflect"
"strconv"
"strings"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/pkg/docker"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/rs/zerolog/log"
)
func (c *Client) listContainerImage(elt model.PrdDocker) []model.Image {
sublog := log.With().
Str("provider", "docker").
Str("id", elt.ID).
Logger()
cli, err := docker.NewClient(elt.Endpoint, elt.ApiVersion, elt.CAFile, elt.CertFile, elt.KeyFile)
if err != nil {
sublog.Error().Err(err).Msg("Cannot create Docker client")
return []model.Image{}
}
ctnFilter := filters.NewArgs()
ctnFilter.Add("status", "running")
if elt.WatchStopped {
ctnFilter.Add("status", "created")
ctnFilter.Add("status", "exited")
}
ctns, err := cli.Containers(ctnFilter)
if err != nil {
sublog.Error().Err(err).Msg("Cannot list Docker containers")
return []model.Image{}
}
var list []model.Image
for _, ctn := range ctns {
image, err := c.containerImage(elt, ctn)
if err != nil {
sublog.Error().Err(err).Msgf("Cannot get image for container %s", ctn.ID)
continue
} else if reflect.DeepEqual(image, model.Image{}) {
sublog.Debug().Msgf("Watch disabled for container %s", ctn.ID)
continue
}
list = append(list, image)
}
return list
}
func (c *Client) containerImage(elt model.PrdDocker, ctn types.Container) (img model.Image, err error) {
img = model.Image{
Name: ctn.Image,
}
if enableStr, ok := ctn.Labels["diun.enable"]; ok {
enable, err := strconv.ParseBool(enableStr)
if err != nil {
return img, fmt.Errorf("cannot parse %s value of label diun.enable", enableStr)
}
if !enable {
return model.Image{}, nil
}
} else if !elt.WatchByDefault {
return model.Image{}, nil
}
for key, value := range ctn.Labels {
switch key {
case "diun.os":
img.Os = value
case "diun.arch":
img.Arch = value
case "diun.regopts_id":
img.RegOptsID = value
case "diun.watch_repo":
if img.WatchRepo, err = strconv.ParseBool(value); err != nil {
return img, fmt.Errorf("cannot parse %s value of label %s", value, key)
}
case "diun.max_tags":
if img.MaxTags, err = strconv.Atoi(value); err != nil {
return img, fmt.Errorf("cannot parse %s value of label %s", value, key)
}
case "diun.include_tags":
img.IncludeTags = strings.Split(value, ";")
case "diun.exclude_tags":
img.ExcludeTags = strings.Split(value, ";")
}
}
return img, nil
}

View file

@ -0,0 +1,47 @@
package docker
import (
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/provider"
"github.com/rs/zerolog/log"
)
// Client represents an active docker provider object
type Client struct {
*provider.Client
elts []model.PrdDocker
}
// New creates new docker provider instance
func New(elts []model.PrdDocker) *provider.Client {
return &provider.Client{Handler: &Client{
elts: elts,
}}
}
// ListJob returns job list to process
func (c *Client) ListJob() []model.Job {
if len(c.elts) == 0 {
return []model.Job{}
}
log.Info().Msgf("Found %d docker provider(s) to analyze...", len(c.elts))
var list []model.Job
for _, elt := range c.elts {
// Swarm mode
if elt.SwarmMode {
continue
}
// Docker
for _, img := range c.listContainerImage(elt) {
list = append(list, model.Job{
Provider: "docker",
ID: elt.ID,
Image: img,
})
}
}
return list
}

View file

@ -0,0 +1,39 @@
package image
import (
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/provider"
"github.com/rs/zerolog/log"
)
// Client represents an active image provider object
type Client struct {
*provider.Client
elts []model.PrdImage
}
// New creates new image provider instance
func New(elts []model.PrdImage) *provider.Client {
return &provider.Client{Handler: &Client{
elts: elts,
}}
}
// ListJob returns job list to process
func (c *Client) ListJob() []model.Job {
if len(c.elts) == 0 {
return []model.Job{}
}
log.Info().Msgf("Found %d image provider(s) to analyze...", len(c.elts))
var list []model.Job
for _, elt := range c.elts {
list = append(list, model.Job{
Provider: "image",
ID: elt.Name,
Image: model.Image(elt),
})
}
return list
}

View file

@ -0,0 +1,16 @@
package provider
import (
"github.com/crazy-max/diun/internal/model"
)
// Handler is a provider interface
type Handler interface {
ListJob() []model.Job
Close() error
}
// Client represents an active provider object
type Client struct {
Handler
}

View file

@ -8,24 +8,36 @@ import (
// Client represents an active docker object
type Client struct {
Api *client.Client
context context.Context
Api *client.Client
}
// NewClient initializes a new Docker API client with default values
func NewClient(endpoint string, apiVersion string, caFile string, certFile string, keyFile string) (*Client, error) {
d, err := client.NewClientWithOpts(
client.WithHost(endpoint),
client.WithVersion(apiVersion),
client.WithTLSClientConfig(caFile, certFile, keyFile),
)
var opts []client.Opt
if endpoint != "" {
opts = append(opts, client.WithHost(endpoint))
}
if apiVersion != "" {
opts = append(opts, client.WithVersion(apiVersion))
}
if caFile != "" && certFile != "" && keyFile != "" {
opts = append(opts, client.WithTLSClientConfig(caFile, certFile, keyFile))
}
cli, err := client.NewClientWithOpts(opts...)
if err != nil {
return nil, err
}
_, err = d.ServerVersion(context.Background())
ctx := context.Background()
_, err = cli.ServerVersion(ctx)
if err != nil {
return nil, err
}
return &Client{Api: d}, err
return &Client{
context: ctx,
Api: cli,
}, err
}

View file

@ -8,15 +8,10 @@ import (
"github.com/docker/docker/api/types/filters"
)
// ContainerOptions holds docker container object options
type ContainerOptions struct {
IncludeStopped bool
}
// ContainerList return container list.
func (c *Client) ContainerList(filterArgs ...filters.KeyValuePair) ([]types.Container, error) {
// Containers return containers based on filters
func (c *Client) Containers(filterArgs filters.Args) ([]types.Container, error) {
containers, err := c.Api.ContainerList(context.Background(), types.ContainerListOptions{
Filters: filters.NewArgs(filterArgs...),
Filters: filterArgs,
})
if err != nil {
return nil, err