vbatts/cri-o
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

*: abstract out netns for multiple platforms

Browse source 
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
This commit is contained in:
Vincent Batts 2018-01-23 10:43:29 -05:00
parent ebdec2ea5b
commit fb87c2f68b
7 changed files with 218 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lib/container_server.go
View file

@ -434,7 +434,7 @@ func (c *ContainerServer) LoadSandbox(id string) error {
return err
}
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs().Path(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
@ -558,7 +558,7 @@ func (c *ContainerServer) LoadContainer(id string) error {
return err
}
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs().Path(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}

129
lib/sandbox/sandbox.go
View file

@ -1,72 +1,18 @@
package sandbox
import (
"crypto/rand"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/fields"
pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
)
// NetNs handles data pertaining a network namespace
type NetNs struct {
sync.Mutex
nn ns.NetNS
symlink *os.File
closed bool
restored bool
}
func (nns *NetNs) symlinkCreate(name string) error {
b := make([]byte, 4)
_, randErr := rand.Reader.Read(b)
if randErr != nil {
return randErr
}
nsName := fmt.Sprintf("%s-%x", name, b)
symlinkPath := filepath.Join(NsRunDir, nsName)
if err := os.Symlink(nns.nn.Path(), symlinkPath); err != nil {
return err
}
fd, err := os.Open(symlinkPath)
if err != nil {
if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
return removeErr
}
return err
}
nns.symlink = fd
return nil
}
func (nns *NetNs) symlinkRemove() error {
if err := nns.symlink.Close(); err != nil {
return fmt.Errorf("failed to close net ns symlink: %v", err)
}
if err := os.RemoveAll(nns.symlink.Name()); err != nil {
return fmt.Errorf("failed to remove net ns symlink: %v", err)
}
}
func isSymbolicLink(path string) (bool, error) {
fi, err := os.Lstat(path)
if err != nil {
@ -78,7 +24,7 @@ func isSymbolicLink(path string) (bool, error) {
// NetNsGet returns the NetNs associated with the given nspath and name
func NetNsGet(nspath, name string) (*NetNs, error) {
if err := ns.IsNSorErr(nspath); err != nil {
if err := isNSorErr(nspath); err != nil {
return nil, ErrClosedNetNS
}
@ -98,13 +44,11 @@ func NetNsGet(nspath, name string) (*NetNs, error) {
resolvedNsPath = nspath
}
netNS, err := ns.GetNS(resolvedNsPath)
netNs, err := getNetNs(resolvedNsPath)
if err != nil {
return nil, err
}
netNs := &NetNs{nn: netNS, closed: false, restored: true}
if symlink {
fd, err := os.Open(nspath)
if err != nil {
@ -123,13 +67,7 @@ func NetNsGet(nspath, name string) (*NetNs, error) {
// HostNetNsPath returns the current network namespace for the host
func HostNetNsPath() (string, error) {
netNS, err := ns.GetCurrentNS()
if err != nil {
return "", err
}
defer netNS.Close()
return netNS.Path(), nil
return hostNetNsPath()
}
// Sandbox contains data surrounding kubernetes sandboxes on the server
@ -386,18 +324,14 @@ func (s *Sandbox) RemoveInfraContainer() {
// NetNs retrieves the network namespace of the sandbox
// If the sandbox uses the host namespace, nil is returned
func (s *Sandbox) NetNs() ns.NetNS {
if s.netns == nil {
return nil
}
return s.netns.nn
func (s *Sandbox) NetNs() *NetNs {
return s.netns
}
// NetNsPath returns the path to the network namespace of the sandbox.
// If the sandbox uses the host namespace, nil is returned
func (s *Sandbox) NetNsPath() string {
if s.netns == nil {
if s.netns == nil || s.netns.symlink == nil {
return ""
}
@ -410,20 +344,16 @@ func (s *Sandbox) NetNsCreate() error {
return fmt.Errorf("net NS already created")
}
netNS, err := ns.NewNS()
netNS, err := newNetNs()
if err != nil {
return err
}
s.netns = &NetNs{
nn: netNS,
closed: false,
}
s.netns = netNS
if err := s.netns.symlinkCreate(s.name); err != nil {
logrus.Warnf("Could not create nentns symlink %v", err)
if err1 := s.netns.nn.Close(); err1 != nil {
if err1 := s.netns.Close(); err1 != nil {
return err1
}
@ -470,44 +400,5 @@ func (s *Sandbox) NetNsRemove() error {
return nil
}
s.netns.Lock()
defer s.netns.Unlock()
if s.netns.closed {
// netNsRemove() can be called multiple
// times without returning an error.
return nil
}
if err := s.netns.symlinkRemove(); err != nil {
return err
}
if err := s.netns.nn.Close(); err != nil {
return err
}
s.netns.closed = true
if s.netns.restored {
// we got namespaces in the form of
// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(s.netns.nn.Path(), "/")
if err != nil {
return err
}
if mounted, err := mount.Mounted(fp); err == nil && mounted {
if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
return err
}
}
if err := os.RemoveAll(s.netns.nn.Path()); err != nil {
return err
}
}
return nil
return s.netns.Remove()
}

151
lib/sandbox/sandbox_linux.go Normal file
View file

@ -0,0 +1,151 @@
// +build linux
package sandbox
import (
"crypto/rand"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/symlink"
"golang.org/x/sys/unix"
)
func isNSorErr(nspath string) error {
return ns.IsNSorErr(nspath)
}
func newNetNs() (*NetNs, error) {
netNS, err := ns.NewNS()
if err != nil {
return nil, err
}
return &NetNs{nn: netNS, closed: false}, nil
}
func getNetNs(path string) (*NetNs, error) {
return &NetNs{}, nil
netNS, err := ns.GetNS(path)
if err != nil {
return nil, err
}
return &NetNs{nn: netNS, closed: false, restored: true}, nil
}
// NetNs handles data pertaining a network namespace
type NetNs struct {
sync.Mutex
nn ns.NetNS
symlink *os.File
closed bool
restored bool
}
func (nns *NetNs) Path() string {
return nns.nn.Path()
}
func (nns *NetNs) Close() error {
return nns.nn.Close()
}
func (nns *NetNs) Remove() error {
nns.Lock()
defer nns.Unlock()
if nns.closed {
// netNsRemove() can be called multiple
// times without returning an error.
return nil
}
if err := nns.symlinkRemove(); err != nil {
return err
}
if err := nns.Close(); err != nil {
return err
}
nns.closed = true
if nns.restored {
// we got namespaces in the form of
// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
// but /var/run on most system is symlinked to /run so we first resolve
// the symlink and then try and see if it's mounted
fp, err := symlink.FollowSymlinkInScope(nns.Path(), "/")
if err != nil {
return err
}
if mounted, err := mount.Mounted(fp); err == nil && mounted {
if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
return err
}
}
if nns.Path() != "" {
if err := os.RemoveAll(nns.Path()); err != nil {
return err
}
}
}
return nil
}
func (nns *NetNs) symlinkCreate(name string) error {
b := make([]byte, 4)
_, randErr := rand.Reader.Read(b)
if randErr != nil {
return randErr
}
nsName := fmt.Sprintf("%s-%x", name, b)
symlinkPath := filepath.Join(NsRunDir, nsName)
if err := os.Symlink(nns.nn.Path(), symlinkPath); err != nil {
return err
}
fd, err := os.Open(symlinkPath)
if err != nil {
if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
return removeErr
}
return err
}
nns.symlink = fd
return nil
}
func (nns *NetNs) symlinkRemove() error {
if err := nns.symlink.Close(); err != nil {
return fmt.Errorf("failed to close net ns symlink: %v", err)
}
if err := os.RemoveAll(nns.symlink.Name()); err != nil {
return fmt.Errorf("failed to remove net ns symlink: %v", err)
}
return nil
}
func hostNetNsPath() (string, error) {
netNS, err := ns.GetCurrentNS()
if err != nil {
return "", err
}
defer netNS.Close()
return netNS.Path(), nil
}

49
lib/sandbox/sandbox_unsupported.go Normal file
View file

@ -0,0 +1,49 @@
// +build !linux
package sandbox
import (
"os"
)
func isNSorErr(nspath string) error {
return nil // TODO(vbatts) ... really not sure ...
}
func newNetNs() (*NetNs, error) {
return &NetNs{}, nil
}
func getNetNs(path string) (*NetNs, error) {
return &NetNs{}, nil
}
// NetNs handles data pertaining a network namespace
// for non-linux this is a noop
type NetNs struct {
symlink *os.File
}
func (nns *NetNs) Path() string {
return ""
}
func (nns *NetNs) symlinkCreate(name string) error {
return nil
}
func (nns *NetNs) symlinkRemove() error {
return nil
}
func (nns *NetNs) Close() error {
return nil
}
func (nns *NetNs) Remove() error {
return nil
}
func hostNetNsPath() (string, error) {
return "", nil // TODO(vbatts) maybe this should have a platform error?
}

9
oci/container.go
View file

@ -9,7 +9,6 @@ import (
"sync"
"time"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/docker/docker/pkg/signal"
specs "github.com/opencontainers/runtime-spec/specs-go"
"k8s.io/apimachinery/pkg/fields"
@ -30,7 +29,7 @@ type Container struct {
crioAnnotations fields.Set
image string
sandbox string
netns ns.NetNS
netns string
terminal bool
stdin bool
stdinOnce bool
@ -71,7 +70,7 @@ type ContainerState struct {
}
// NewContainer creates a container object.
func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
func NewContainer(id string, name string, bundlePath string, logPath string, netns string, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
state := &ContainerState{}
state.Created = created
c := &Container{
@ -223,11 +222,11 @@ func (c *Container) NetNsPath() (string, error) {
return "", fmt.Errorf("container state is not populated")
}
if c.netns == nil {
if c.netns == "" {
return fmt.Sprintf("/proc/%d/ns/net", c.state.Pid), nil
}
return c.netns.Path(), nil
return c.netns, nil
}
// Metadata returns the metadata of the container.

2
server/container_create.go
View file

@ -1303,7 +1303,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
crioAnnotations := specgen.Spec().Annotations
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs().Path(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
if err != nil {
return nil, err
}

2
server/sandbox_run.go
View file

@ -513,7 +513,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
g.AddAnnotation(annotations.HostnamePath, hostnamePath)
sb.AddHostnamePath(hostnamePath)
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs().Path(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
if err != nil {
return nil, err
}