From d168fc5fec4929f1b2746914a235c0fe73adb24e Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Fri, 1 Dec 2017 17:48:14 +0100 Subject: [PATCH] lib: sandbox: refactor to memory store Signed-off-by: Antonio Murdaca --- lib/container_server.go | 44 ++++++++---------- lib/sandbox/history.go | 31 +++++++++++++ lib/sandbox/memory_store.go | 92 +++++++++++++++++++++++++++++++++++++ lib/sandbox/sandbox.go | 3 ++ lib/sandbox/store.go | 27 +++++++++++ 5 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 lib/sandbox/history.go create mode 100644 lib/sandbox/memory_store.go create mode 100644 lib/sandbox/store.go diff --git a/lib/container_server.go b/lib/container_server.go index fdc9f9b7..7c9c987d 100644 --- a/lib/container_server.go +++ b/lib/container_server.go @@ -168,7 +168,7 @@ func New(config *Config) (*ContainerServer, error) { state: &containerServerState{ containers: oci.NewMemoryStore(), infraContainers: oci.NewMemoryStore(), - sandboxes: make(map[string]*sandbox.Sandbox), + sandboxes: sandbox.NewMemoryStore(), processLevels: make(map[string]int), }, config: config, @@ -617,7 +617,7 @@ func (c *ContainerServer) Shutdown() error { type containerServerState struct { containers oci.ContainerStorer infraContainers oci.ContainerStorer - sandboxes map[string]*sandbox.Sandbox + sandboxes sandbox.Storer // processLevels The number of sandboxes using the same SELinux MCS level. Need to release MCS Level, when count reaches 0 processLevels map[string]int } @@ -626,7 +626,7 @@ type containerServerState struct { func (c *ContainerServer) AddContainer(ctr *oci.Container) { c.stateLock.Lock() defer c.stateLock.Unlock() - sandbox := c.state.sandboxes[ctr.Sandbox()] + sandbox := c.state.sandboxes.Get(ctr.Sandbox()) sandbox.AddContainer(ctr) c.state.containers.Add(ctr.ID(), ctr) } @@ -665,7 +665,7 @@ func (c *ContainerServer) RemoveContainer(ctr *oci.Container) { c.stateLock.Lock() defer c.stateLock.Unlock() sbID := ctr.Sandbox() - sb := c.state.sandboxes[sbID] + sb := c.state.sandboxes.Get(sbID) sb.RemoveContainer(ctr) c.state.containers.Delete(ctr.ID()) } @@ -706,7 +706,7 @@ func (c *ContainerServer) ListContainers(filters ...func(*oci.Container) bool) ( func (c *ContainerServer) AddSandbox(sb *sandbox.Sandbox) { c.stateLock.Lock() defer c.stateLock.Unlock() - c.state.sandboxes[sb.ID()] = sb + c.state.sandboxes.Add(sb.ID(), sb) c.state.processLevels[selinux.NewContext(sb.ProcessLabel())["level"]]++ } @@ -714,17 +714,14 @@ func (c *ContainerServer) AddSandbox(sb *sandbox.Sandbox) { func (c *ContainerServer) GetSandbox(id string) *sandbox.Sandbox { c.stateLock.Lock() defer c.stateLock.Unlock() - return c.state.sandboxes[id] + return c.state.sandboxes.Get(id) } // GetSandboxContainer returns a sandbox's infra container func (c *ContainerServer) GetSandboxContainer(id string) *oci.Container { c.stateLock.Lock() defer c.stateLock.Unlock() - sb, ok := c.state.sandboxes[id] - if !ok { - return nil - } + sb := c.state.sandboxes.Get(id) return sb.InfraContainer() } @@ -732,21 +729,25 @@ func (c *ContainerServer) GetSandboxContainer(id string) *oci.Container { func (c *ContainerServer) HasSandbox(id string) bool { c.stateLock.Lock() defer c.stateLock.Unlock() - _, ok := c.state.sandboxes[id] - return ok + sb := c.state.sandboxes.Get(id) + return sb != nil } // RemoveSandbox removes a sandbox from the state store func (c *ContainerServer) RemoveSandbox(id string) { c.stateLock.Lock() defer c.stateLock.Unlock() - processLabel := c.state.sandboxes[id].ProcessLabel() - delete(c.state.sandboxes, id) + sb := c.state.sandboxes.Get(id) + processLabel := sb.ProcessLabel() + c.state.sandboxes.Delete(id) level := selinux.NewContext(processLabel)["level"] - c.state.processLevels[level]-- - if c.state.processLevels[level] == 0 { - label.ReleaseLabel(processLabel) - delete(c.state.processLevels, level) + pl, ok := c.state.processLevels[level] + if ok { + c.state.processLevels[level] = pl - 1 + if c.state.processLevels[level] == 0 { + label.ReleaseLabel(processLabel) + delete(c.state.processLevels, level) + } } } @@ -754,12 +755,7 @@ func (c *ContainerServer) RemoveSandbox(id string) { func (c *ContainerServer) ListSandboxes() []*sandbox.Sandbox { c.stateLock.Lock() defer c.stateLock.Unlock() - sbArray := make([]*sandbox.Sandbox, 0, len(c.state.sandboxes)) - for _, sb := range c.state.sandboxes { - sbArray = append(sbArray, sb) - } - - return sbArray + return c.state.sandboxes.List() } // LibcontainerStats gets the stats for the container with the given id from runc/libcontainer diff --git a/lib/sandbox/history.go b/lib/sandbox/history.go new file mode 100644 index 00000000..84d0291d --- /dev/null +++ b/lib/sandbox/history.go @@ -0,0 +1,31 @@ +package sandbox + +import "sort" + +// History is a convenience type for storing a list of sandboxes, +// sorted by creation date in descendant order. +type History []*Sandbox + +// Len returns the number of sandboxes in the history. +func (history *History) Len() int { + return len(*history) +} + +// Less compares two sandboxes and returns true if the second one +// was created before the first one. +func (history *History) Less(i, j int) bool { + sandboxes := *history + // FIXME: state access should be serialized + return sandboxes[j].created.Before(sandboxes[i].created) +} + +// Swap switches sandboxes i and j positions in the history. +func (history *History) Swap(i, j int) { + sandboxes := *history + sandboxes[i], sandboxes[j] = sandboxes[j], sandboxes[i] +} + +// sort orders the history by creation date in descendant order. +func (history *History) sort() { + sort.Sort(history) +} diff --git a/lib/sandbox/memory_store.go b/lib/sandbox/memory_store.go new file mode 100644 index 00000000..6bf0cd84 --- /dev/null +++ b/lib/sandbox/memory_store.go @@ -0,0 +1,92 @@ +package sandbox + +import "sync" + +// memoryStore implements a Store in memory. +type memoryStore struct { + s map[string]*Sandbox + sync.RWMutex +} + +// NewMemoryStore initializes a new memory store. +func NewMemoryStore() Storer { + return &memoryStore{ + s: make(map[string]*Sandbox), + } +} + +// Add appends a new sandbox to the memory store. +// It overrides the id if it existed before. +func (c *memoryStore) Add(id string, cont *Sandbox) { + c.Lock() + c.s[id] = cont + c.Unlock() +} + +// Get returns a sandbox from the store by id. +func (c *memoryStore) Get(id string) *Sandbox { + c.RLock() + res := c.s[id] + c.RUnlock() + return res +} + +// Delete removes a sandbox from the store by id. +func (c *memoryStore) Delete(id string) { + c.Lock() + delete(c.s, id) + c.Unlock() +} + +// List returns a sorted list of sandboxes from the store. +// The sandboxes are ordered by creation date. +func (c *memoryStore) List() []*Sandbox { + sandboxes := History(c.all()) + sandboxes.sort() + return sandboxes +} + +// Size returns the number of sandboxes in the store. +func (c *memoryStore) Size() int { + c.RLock() + defer c.RUnlock() + return len(c.s) +} + +// First returns the first sandbox found in the store by a given filter. +func (c *memoryStore) First(filter StoreFilter) *Sandbox { + for _, cont := range c.all() { + if filter(cont) { + return cont + } + } + return nil +} + +// ApplyAll calls the reducer function with every sandbox in the store. +// This operation is asynchronous in the memory store. +// NOTE: Modifications to the store MUST NOT be done by the StoreReducer. +func (c *memoryStore) ApplyAll(apply StoreReducer) { + wg := new(sync.WaitGroup) + for _, cont := range c.all() { + wg.Add(1) + go func(sandbox *Sandbox) { + apply(sandbox) + wg.Done() + }(cont) + } + + wg.Wait() +} + +func (c *memoryStore) all() []*Sandbox { + c.RLock() + sandboxes := make([]*Sandbox, 0, len(c.s)) + for _, cont := range c.s { + sandboxes = append(sandboxes, cont) + } + c.RUnlock() + return sandboxes +} + +var _ Storer = &memoryStore{} diff --git a/lib/sandbox/sandbox.go b/lib/sandbox/sandbox.go index 7293747b..7624b072 100644 --- a/lib/sandbox/sandbox.go +++ b/lib/sandbox/sandbox.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/containernetworking/plugins/pkg/ns" "github.com/docker/docker/pkg/mount" @@ -158,6 +159,7 @@ type Sandbox struct { // ipv4 or ipv6 cache ip string seccompProfilePath string + created time.Time } const ( @@ -202,6 +204,7 @@ func New(id, namespace, name, kubeName, logDir string, labels, annotations map[s sb.resolvPath = resolvPath sb.hostname = hostname sb.portMappings = portMappings + sb.created = time.Now() return sb, nil } diff --git a/lib/sandbox/store.go b/lib/sandbox/store.go new file mode 100644 index 00000000..83d705cd --- /dev/null +++ b/lib/sandbox/store.go @@ -0,0 +1,27 @@ +package sandbox + +// StoreFilter defines a function to filter +// sandboxes in the store. +type StoreFilter func(*Sandbox) bool + +// StoreReducer defines a function to +// manipulate sandboxes in the store +type StoreReducer func(*Sandbox) + +// Storer defines an interface that any container store must implement. +type Storer interface { + // Add appends a new sandbox to the store. + Add(string, *Sandbox) + // Get returns a sandbox from the store by the identifier it was stored with. + Get(string) *Sandbox + // Delete removes a sandbox from the store by the identifier it was stored with. + Delete(string) + // List returns a list of sandboxes from the store. + List() []*Sandbox + // Size returns the number of sandboxes in the store. + Size() int + // First returns the first sandbox found in the store by a given filter. + First(StoreFilter) *Sandbox + // ApplyAll calls the reducer function with every sandbox in the store. + ApplyAll(StoreReducer) +}