diff --git a/README.md b/README.md index 14cc3014..d79bc424 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ For sync communication we have an IRC channel #CRI-O, on chat.freenode.net, that - socat - iproute - iptables +- strace (required for debugging using cri-o strace annotation) Latest version of `runc` is expected to be installed on the system. It is picked up as the default runtime by CRI-O. diff --git a/cmd/crio/main.go b/cmd/crio/main.go index a058f296..ba79226d 100644 --- a/cmd/crio/main.go +++ b/cmd/crio/main.go @@ -147,6 +147,9 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error { if ctx.GlobalIsSet("image-volumes") { config.ImageVolumes = lib.ImageVolumesType(ctx.GlobalString("image-volumes")) } + if ctx.GlobalIsSet("allow-strace") { + config.AllowStrace = ctx.GlobalBool("allow-strace") + } return nil } @@ -351,6 +354,10 @@ func main() { Value: 9090, Usage: "port for the metrics endpoint", }, + cli.BoolFlag{ + Name: "allow-strace", + Usage: "allow the cri-o strace annotation/label to take effect", + }, } sort.Sort(cli.FlagsByName(app.Flags)) diff --git a/docs/crio.8.md b/docs/crio.8.md index 8408978b..de483e7a 100644 --- a/docs/crio.8.md +++ b/docs/crio.8.md @@ -120,6 +120,8 @@ crio [GLOBAL OPTIONS] config [OPTIONS] **--cpu-profile**: Set the CPU profile file path +**--allow-strace**: Allow strace for containers when the strace annotation/label "io.kubernetes.cri-o.Strace=true" is set + **--version, -v**: Print the version # COMMANDS diff --git a/lib/config.go b/lib/config.go index 6a63b2b0..4bd5075c 100644 --- a/lib/config.go +++ b/lib/config.go @@ -168,6 +168,10 @@ type RuntimeConfig struct { // ContainerExitsDir is the directory in which container exit files are // written to by conmon. ContainerExitsDir string `toml:"container_exits_dir"` + + // AllowStrace determinates whether strace is started for a container + // which has the CRI-O strace annotation/label "io.kubernetes.cri-o.Strace=true". + AllowStrace bool `toml:"allow_strace"` } // ImageConfig represents the "crio.image" TOML config table. diff --git a/lib/container_server.go b/lib/container_server.go index 9a4704b7..071a8809 100644 --- a/lib/container_server.go +++ b/lib/container_server.go @@ -389,7 +389,7 @@ func (c *ContainerServer) LoadSandbox(id string) error { return err } - scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) + scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, c.Config().AllowStrace, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) if err != nil { return err } @@ -513,7 +513,7 @@ func (c *ContainerServer) LoadContainer(id string) error { return err } - ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) + ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), c.Config().AllowStrace, containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"]) if err != nil { return err } diff --git a/oci/container.go b/oci/container.go index c71152bf..536c9114 100644 --- a/oci/container.go +++ b/oci/container.go @@ -36,6 +36,7 @@ type Container struct { stdinOnce bool privileged bool trusted bool + allowStrace bool state *ContainerState metadata *pb.ContainerMetadata opLock sync.Locker @@ -71,7 +72,7 @@ type ContainerState struct { } // NewContainer creates a container object. -func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) { +func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, allowStrace bool, dir string, created time.Time, stopSignal string) (*Container, error) { state := &ContainerState{} state.Created = created c := &Container{ @@ -87,6 +88,7 @@ func NewContainer(id string, name string, bundlePath string, logPath string, net stdinOnce: stdinOnce, privileged: privileged, trusted: trusted, + allowStrace: allowStrace, metadata: metadata, annotations: annotations, crioAnnotations: crioAnnotations, diff --git a/oci/oci.go b/oci/oci.go index 658079a3..b4226c6f 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -14,6 +14,7 @@ import ( "time" "github.com/containerd/cgroups" + "github.com/kubernetes-incubator/cri-o/pkg/annotations" "github.com/kubernetes-incubator/cri-o/utils" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -284,6 +285,7 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) (err error) if ss.err != nil { return fmt.Errorf("error reading container (probably exited) json message: %v", ss.err) } + c.state.Pid = ss.si.Pid logrus.Debugf("Received container pid: %d", ss.si.Pid) if ss.si.Pid == -1 { if ss.si.Message != "" { @@ -297,6 +299,29 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) (err error) logrus.Errorf("Container creation timeout (%v)", ContainerCreateTimeout) return fmt.Errorf("create container timeout") } + + enableStrace := false + if _, ok := c.annotations[annotations.Strace]; ok { + logrus.Debugf("Enabling strace from annotation") + enableStrace = true + } + if !enableStrace { + if _, ok := c.labels[annotations.Strace]; ok { + logrus.Debugf("Enabling strace from label") + enableStrace = true + } + } + + if enableStrace && c.allowStrace { + go func() { + straceCmd := exec.Command("strace", "-f", "-o", fmt.Sprintf("/tmp/%v", c.id), "-p", fmt.Sprintf("%d", c.state.Pid)) + _, err := straceCmd.CombinedOutput() + if err != nil { + logrus.Errorf("Failed to execute strace: %v", err) + } + }() + } + return nil } diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 9b5b1352..966952a4 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -84,6 +84,9 @@ const ( // Volumes is the volumes annotatoin Volumes = "io.kubernetes.cri-o.Volumes" + + // Strace is enable strace debug annotation. + Strace = "io.kubernetes.cri-o.Strace" ) // ContainerType values diff --git a/server/container_create.go b/server/container_create.go index 6afc412b..bb3961d3 100644 --- a/server/container_create.go +++ b/server/container_create.go @@ -1252,7 +1252,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string, crioAnnotations := specgen.Spec().Annotations - container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal) + container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), s.config.Config.AllowStrace, containerInfo.Dir, created, containerImageConfig.Config.StopSignal) if err != nil { return nil, err } diff --git a/server/inspect_test.go b/server/inspect_test.go index 7246ef86..406ece3d 100644 --- a/server/inspect_test.go +++ b/server/inspect_test.go @@ -67,7 +67,7 @@ func TestGetContainerInfo(t *testing.T) { "io.kubernetes.test1": "value1", } getContainerFunc := func(id string) *oci.Container { - container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "image", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL") + container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "image", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, false, "/root/for/container", created, "SIGKILL") if err != nil { t.Fatal(err) } @@ -184,7 +184,7 @@ func TestGetContainerInfoCtrStateNil(t *testing.T) { labels := map[string]string{} annotations := map[string]string{} getContainerFunc := func(id string) *oci.Container { - container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL") + container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, false, "/root/for/container", created, "SIGKILL") if err != nil { t.Fatal(err) } @@ -215,7 +215,7 @@ func TestGetContainerInfoSandboxNotFound(t *testing.T) { labels := map[string]string{} annotations := map[string]string{} getContainerFunc := func(id string) *oci.Container { - container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL") + container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, false, "/root/for/container", created, "SIGKILL") if err != nil { t.Fatal(err) } diff --git a/server/sandbox_run.go b/server/sandbox_run.go index 0cf40ea8..373ff0a9 100644 --- a/server/sandbox_run.go +++ b/server/sandbox_run.go @@ -495,7 +495,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest g.AddAnnotation(annotations.HostnamePath, hostnamePath) sb.AddHostnamePath(hostnamePath) - container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal) + container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), s.config.Config.AllowStrace, podContainer.Dir, created, podContainer.Config.Config.StopSignal) if err != nil { return nil, err }