Merge branch 'master' of github.com:kubernetes-incubator/cri-o into umount

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
This commit is contained in:
Daniel J Walsh 2017-09-26 21:41:02 +00:00
commit 8e6d5b1c9a
42 changed files with 1456 additions and 1021 deletions

View file

@ -4,7 +4,7 @@
[![Build Status](https://img.shields.io/travis/kubernetes-incubator/cri-o.svg?maxAge=2592000&style=flat-square)](https://travis-ci.org/kubernetes-incubator/cri-o)
[![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes-incubator/cri-o?style=flat-square)](https://goreportcard.com/report/github.com/kubernetes-incubator/cri-o)
### Status: Release Candidate 1
### Status: Release Candidate 2
## What is the scope of this project?

103
client/client.go Normal file
View file

@ -0,0 +1,103 @@
package client
import (
"encoding/json"
"fmt"
"net"
"net/http"
"syscall"
"time"
"github.com/kubernetes-incubator/cri-o/types"
)
const (
maxUnixSocketPathSize = len(syscall.RawSockaddrUnix{}.Path)
)
// CrioClient is an interface to get information from crio daemon endpoint.
type CrioClient interface {
DaemonInfo() (types.CrioInfo, error)
ContainerInfo(string) (*types.ContainerInfo, error)
}
type crioClientImpl struct {
client *http.Client
crioSocketPath string
}
func configureUnixTransport(tr *http.Transport, proto, addr string) error {
if len(addr) > maxUnixSocketPathSize {
return fmt.Errorf("Unix socket path %q is too long", addr)
}
// No need for compression in local communications.
tr.DisableCompression = true
tr.Dial = func(_, _ string) (net.Conn, error) {
return net.DialTimeout(proto, addr, 32*time.Second)
}
return nil
}
// New returns a crio client
func New(crioSocketPath string) (CrioClient, error) {
tr := new(http.Transport)
configureUnixTransport(tr, "unix", crioSocketPath)
c := &http.Client{
Transport: tr,
}
return &crioClientImpl{
client: c,
crioSocketPath: crioSocketPath,
}, nil
}
func (c *crioClientImpl) getRequest(path string) (*http.Request, error) {
req, err := http.NewRequest("GET", path, nil)
if err != nil {
return nil, err
}
// For local communications over a unix socket, it doesn't matter what
// the host is. We just need a valid and meaningful host name.
req.Host = "crio"
req.URL.Host = c.crioSocketPath
req.URL.Scheme = "http"
return req, nil
}
// DaemonInfo return cri-o daemon info from the cri-o
// info endpoint.
func (c *crioClientImpl) DaemonInfo() (types.CrioInfo, error) {
info := types.CrioInfo{}
req, err := c.getRequest("/info")
if err != nil {
return info, err
}
resp, err := c.client.Do(req)
if err != nil {
return info, err
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
return info, err
}
return info, nil
}
// ContainerInfo returns container info by querying
// the cri-o container endpoint.
func (c *crioClientImpl) ContainerInfo(id string) (*types.ContainerInfo, error) {
req, err := c.getRequest("/containers/" + id)
if err != nil {
return nil, err
}
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
cInfo := types.ContainerInfo{}
if err := json.NewDecoder(resp.Body).Decode(&cInfo); err != nil {
return nil, err
}
return &cInfo, nil
}

View file

@ -108,6 +108,10 @@ hooks_dir_path = "{{ .HooksDirPath }}"
# pids_limit is the number of processes allowed in a container
pids_limit = {{ .PidsLimit }}
# log_size_max is the max limit for the container log size in bytes.
# Negative values indicate that no limit is imposed.
log_size_max = {{ .LogSizeMax }}
# The "crio.image" table contains settings pertaining to the
# management of OCI images.
[crio.image]

View file

@ -125,6 +125,9 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error {
if ctx.GlobalIsSet("pids-limit") {
config.PidsLimit = ctx.GlobalInt64("pids-limit")
}
if ctx.GlobalIsSet("log-size-max") {
config.LogSizeMax = ctx.GlobalInt64("log-size-max")
}
if ctx.GlobalIsSet("cni-config-dir") {
config.NetworkDir = ctx.GlobalString("cni-config-dir")
}
@ -289,6 +292,11 @@ func main() {
Value: libkpod.DefaultPidsLimit,
Usage: "maximum number of processes allowed in a container",
},
cli.Int64Flag{
Name: "log-size-max",
Value: libkpod.DefaultLogSizeMax,
Usage: "maximum log size in bytes for a container",
},
cli.StringFlag{
Name: "cni-config-dir",
Usage: "CNI configuration files directory",

View file

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"log"
"net/url"
@ -8,6 +9,7 @@ import (
"strings"
"time"
"github.com/kubernetes-incubator/cri-o/client"
"github.com/urfave/cli"
"golang.org/x/net/context"
remocommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
@ -21,6 +23,7 @@ var containerCommand = cli.Command{
Aliases: []string{"ctr"},
Subcommands: []cli.Command{
createContainerCommand,
inspectContainerCommand,
startContainerCommand,
stopContainerCommand,
removeContainerCommand,
@ -617,3 +620,37 @@ func ListContainers(client pb.RuntimeServiceClient, opts listOptions) error {
}
return nil
}
var inspectContainerCommand = cli.Command{
Name: "inspect",
Usage: "get container info from crio daemon",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Value: "",
Usage: "id of the container",
},
},
Action: func(context *cli.Context) error {
ID := context.String("id")
if ID == "" {
return fmt.Errorf("ID cannot be empty")
}
c, err := client.New(context.GlobalString("connect"))
if err != nil {
return err
}
cInfo, err := c.ContainerInfo(ID)
if err != nil {
return err
}
jsonBytes, err := json.MarshalIndent(cInfo, "", " ")
if err != nil {
return err
}
fmt.Println(string(jsonBytes))
return nil
},
}

31
cmd/crioctl/info.go Normal file
View file

@ -0,0 +1,31 @@
package main
import (
"encoding/json"
"fmt"
"github.com/kubernetes-incubator/cri-o/client"
"github.com/urfave/cli"
)
var infoCommand = cli.Command{
Name: "info",
Usage: "get crio daemon info",
Action: func(context *cli.Context) error {
c, err := client.New(context.GlobalString("connect"))
if err != nil {
return err
}
di, err := c.DaemonInfo()
if err != nil {
return err
}
jsonBytes, err := json.MarshalIndent(di, "", " ")
if err != nil {
return err
}
fmt.Println(string(jsonBytes))
return nil
},
}

View file

@ -91,6 +91,7 @@ func main() {
containerCommand,
runtimeVersionCommand,
imageCommand,
infoCommand,
}
app.Flags = []cli.Flag{

View file

@ -7,6 +7,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <sys/prctl.h>
#include <sys/socket.h>
#include <sys/types.h>
@ -20,6 +21,7 @@
#include <termios.h>
#include <syslog.h>
#include <unistd.h>
#include <inttypes.h>
#include <glib.h>
#include <glib-unix.h>
@ -107,6 +109,7 @@ static bool opt_exec = false;
static char *opt_log_path = NULL;
static char *opt_exit_dir = NULL;
static int opt_timeout = 0;
static int64_t opt_log_size_max = -1;
static GOptionEntry opt_entries[] =
{
{ "terminal", 't', 0, G_OPTION_ARG_NONE, &opt_terminal, "Terminal", NULL },
@ -122,6 +125,7 @@ static GOptionEntry opt_entries[] =
{ "exit-dir", 0, 0, G_OPTION_ARG_STRING, &opt_exit_dir, "Path to the directory where exit files are written", NULL },
{ "log-path", 'l', 0, G_OPTION_ARG_STRING, &opt_log_path, "Log file path", NULL },
{ "timeout", 'T', 0, G_OPTION_ARG_INT, &opt_timeout, "Timeout in seconds", NULL },
{ "log-size-max", 0, 0, G_OPTION_ARG_INT64, &opt_log_size_max, "Maximum size of log file", NULL },
{ NULL }
};
@ -130,6 +134,8 @@ static GOptionEntry opt_entries[] =
#define CGROUP_ROOT "/sys/fs/cgroup"
static int log_fd = -1;
static ssize_t write_all(int fd, const void *buf, size_t count)
{
size_t remaining = count;
@ -281,11 +287,13 @@ const char *stdpipe_name(stdpipe_t pipe)
* line in buf, and will partially write the final line of the log if buf is
* not terminated by a newline.
*/
int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen)
static int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen)
{
char tsbuf[TSBUFLEN];
static stdpipe_t trailing_line = NO_PIPE;
writev_buffer_t bufv = {0};
static int64_t bytes_written = 0;
int64_t bytes_to_be_written = 0;
/*
* Use the same timestamp for every line of the log in this buffer.
@ -299,6 +307,8 @@ int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen)
while (buflen > 0) {
const char *line_end = NULL;
ptrdiff_t line_len = 0;
bool insert_newline = FALSE;
bool insert_timestamp = FALSE;
/* Find the end of the line, or alternatively the end of the buffer. */
line_end = memchr(buf, '\n', buflen);
@ -306,12 +316,15 @@ int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen)
line_end = &buf[buflen-1];
line_len = line_end - buf + 1;
/*
* Write the (timestamp, stream) tuple if there isn't any trailing
* output from the previous line (or if there is trailing output but
* the current buffer being printed is from a different pipe).
*/
bytes_to_be_written = line_len;
if (trailing_line != pipe) {
/*
* Write the (timestamp, stream) tuple if there isn't any trailing
* output from the previous line (or if there is trailing output but
* the current buffer being printed is from a different pipe).
*/
insert_timestamp = TRUE;
bytes_to_be_written += (TSBUFLEN - 1);
/*
* If there was a trailing line from a different pipe, prepend a
* newline to split it properly. This technically breaks the flow
@ -319,9 +332,49 @@ int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen)
* wasn't one output) but without modifying the file in a
* non-append-only way there's not much we can do.
*/
if ((trailing_line != NO_PIPE &&
writev_buffer_append_segment(fd, &bufv, "\n", -1) < 0) ||
writev_buffer_append_segment(fd, &bufv, tsbuf, -1) < 0) {
if (trailing_line != NO_PIPE) {
insert_newline = TRUE;
bytes_to_be_written += 1;
}
}
/*
* We re-open the log file if writing out the bytes will exceed the max
* log size. We also reset the state so that the new file is started with
* a timestamp.
*/
if ((opt_log_size_max > 0) && (bytes_written + bytes_to_be_written) > opt_log_size_max) {
ninfo("Creating new log file");
insert_newline = FALSE;
insert_timestamp = TRUE;
bytes_written = 0;
/* Close the existing fd */
close(fd);
/* Unlink the file */
if (unlink(opt_log_path) < 0) {
pexit("Failed to unlink log file");
}
/* Open the log path file again */
log_fd = open(opt_log_path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, 0600);
if (log_fd < 0)
pexit("Failed to open log file");
fd = log_fd;
}
/* Output a newline */
if (insert_newline) {
if (writev_buffer_append_segment(fd, &bufv, "\n", -1) < 0) {
nwarn("failed to write newline to log");
goto next;
}
}
/* Output a timestamp */
if (insert_timestamp) {
if (writev_buffer_append_segment(fd, &bufv, tsbuf, -1) < 0) {
nwarn("failed to write (timestamp, stream) to log");
goto next;
}
@ -333,6 +386,8 @@ int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen)
goto next;
}
bytes_written += bytes_to_be_written;
/* If we did not output a full line, then we are a trailing_line. */
trailing_line = (*line_end == '\n') ? NO_PIPE : pipe;
@ -346,6 +401,8 @@ next:
nwarn("failed to flush buffer to log");
}
ninfo("Total bytes written: %"PRId64"", bytes_written);
return 0;
}
@ -481,7 +538,6 @@ static int conn_sock = -1;
static int conn_sock_readable;
static int conn_sock_writable;
static int log_fd = -1;
static int oom_event_fd = -1;
static int attach_socket_fd = -1;
static int console_socket_fd = -1;

View file

@ -105,6 +105,9 @@ set the CPU profile file path
**--log-format**=""
Set the format used by logs ('text' (default), or 'json') (default: "text")
**--log-size-max**=""
Maximum log size in bytes for a container (default: -1 (no limit))
**--pause-command**=""
Path to the pause executable in the pause image (default: "/pause")

View file

@ -54,6 +54,11 @@ The `crio` table supports the following options:
**conmon_env**=[]
Environment variable list for conmon process (default: ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",])
**log_size_max**=""
Maximum sized allowed for the container log file (default: -1)
Negative numbers indicate that no size limit is imposed.
The file is truncated and re-opened so the limit is never exceeded.
**pids_limit**=""
Maximum number of processes allowed in a container (default: 1024)

View file

@ -51,6 +51,10 @@ const (
// DefaultPidsLimit is the default value for maximum number of processes
// allowed inside a container
DefaultPidsLimit = 1024
// DefaultLogSizeMax is the default value for the maximum log size
// allowed for a container. Negative values mean that no limit is imposed.
DefaultLogSizeMax = -1
)
// This structure is necessary to fake the TOML tables when parsing,
@ -145,6 +149,12 @@ type RuntimeConfig struct {
// by the cgroup process number controller.
PidsLimit int64 `toml:"pids_limit"`
// LogSizeMax is the maximum number of bytes after which the log file
// will be truncated. It can be expressed as a human-friendly string
// that is parsed to bytes.
// Negative values indicate that the log file won't be truncated.
LogSizeMax int64 `toml:"log_size_max"`
// ContainerExitsDir is the directory in which container exit files are
// written to by conmon.
ContainerExitsDir string `toml:"container_exits_dir"`
@ -274,6 +284,7 @@ func DefaultConfig() *Config {
PidsLimit: DefaultPidsLimit,
ContainerExitsDir: containerExitsDir,
HooksDirPath: DefaultHooksDirPath,
LogSizeMax: DefaultLogSizeMax,
},
ImageConfig: ImageConfig{
DefaultTransport: defaultTransport,

View file

@ -121,7 +121,7 @@ func New(config *Config) (*ContainerServer, error) {
return nil, err
}
runtime, err := oci.New(config.Runtime, config.RuntimeUntrustedWorkload, config.DefaultWorkloadTrust, config.Conmon, config.ConmonEnv, config.CgroupManager, config.ContainerExitsDir)
runtime, err := oci.New(config.Runtime, config.RuntimeUntrustedWorkload, config.DefaultWorkloadTrust, config.Conmon, config.ConmonEnv, config.CgroupManager, config.ContainerExitsDir, config.LogSizeMax)
if err != nil {
return nil, err
}
@ -384,7 +384,7 @@ func (c *ContainerServer) LoadSandbox(id string) error {
return err
}
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
@ -507,7 +507,7 @@ func (c *ContainerServer) LoadContainer(id string) error {
return err
}
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}

View file

@ -22,22 +22,23 @@ const (
// Container represents a runtime container.
type Container struct {
id string
name string
logPath string
labels fields.Set
annotations fields.Set
image string
sandbox string
netns ns.NetNS
terminal bool
stdin bool
stdinOnce bool
privileged bool
trusted bool
state *ContainerState
metadata *pb.ContainerMetadata
opLock sync.Locker
id string
name string
logPath string
labels fields.Set
annotations fields.Set
crioAnnotations fields.Set
image string
sandbox string
netns ns.NetNS
terminal bool
stdin bool
stdinOnce bool
privileged bool
trusted bool
state *ContainerState
metadata *pb.ContainerMetadata
opLock sync.Locker
// this is the /var/run/storage/... directory, erased on reboot
bundlePath string
// this is the /var/lib/storage/... directory
@ -68,31 +69,32 @@ type ContainerState struct {
}
// NewContainer creates a container object.
func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
state := &ContainerState{}
state.Created = created
c := &Container{
id: id,
name: name,
bundlePath: bundlePath,
logPath: logPath,
labels: labels,
sandbox: sandbox,
netns: netns,
terminal: terminal,
stdin: stdin,
stdinOnce: stdinOnce,
privileged: privileged,
trusted: trusted,
metadata: metadata,
annotations: annotations,
image: image,
imageName: imageName,
imageRef: imageRef,
dir: dir,
state: state,
stopSignal: stopSignal,
opLock: new(sync.Mutex),
id: id,
name: name,
bundlePath: bundlePath,
logPath: logPath,
labels: labels,
sandbox: sandbox,
netns: netns,
terminal: terminal,
stdin: stdin,
stdinOnce: stdinOnce,
privileged: privileged,
trusted: trusted,
metadata: metadata,
annotations: annotations,
crioAnnotations: crioAnnotations,
image: image,
imageName: imageName,
imageRef: imageRef,
dir: dir,
state: state,
stopSignal: stopSignal,
opLock: new(sync.Mutex),
}
return c, nil
}
@ -163,6 +165,11 @@ func (c *Container) Annotations() map[string]string {
return c.annotations
}
// CrioAnnotations returns the crio annotations of the container.
func (c *Container) CrioAnnotations() map[string]string {
return c.crioAnnotations
}
// Image returns the image of the container.
func (c *Container) Image() string {
return c.image

View file

@ -38,7 +38,7 @@ const (
)
// New creates a new Runtime with options provided
func New(runtimeTrustedPath string, runtimeUntrustedPath string, trustLevel string, conmonPath string, conmonEnv []string, cgroupManager string, containerExitsDir string) (*Runtime, error) {
func New(runtimeTrustedPath string, runtimeUntrustedPath string, trustLevel string, conmonPath string, conmonEnv []string, cgroupManager string, containerExitsDir string, logSizeMax int64) (*Runtime, error) {
r := &Runtime{
name: filepath.Base(runtimeTrustedPath),
trustedPath: runtimeTrustedPath,
@ -48,6 +48,7 @@ func New(runtimeTrustedPath string, runtimeUntrustedPath string, trustLevel stri
conmonEnv: conmonEnv,
cgroupManager: cgroupManager,
containerExitsDir: containerExitsDir,
logSizeMax: logSizeMax,
}
return r, nil
}
@ -62,6 +63,7 @@ type Runtime struct {
conmonEnv []string
cgroupManager string
containerExitsDir string
logSizeMax int64
}
// syncInfo is used to return data from monitor process to daemon
@ -156,6 +158,9 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) error {
args = append(args, "-p", filepath.Join(c.bundlePath, "pidfile"))
args = append(args, "-l", c.logPath)
args = append(args, "--exit-dir", r.containerExitsDir)
if r.logSizeMax >= 0 {
args = append(args, "--log-size-max", fmt.Sprintf("%v", r.logSizeMax))
}
if c.terminal {
args = append(args, "-t")
} else if c.stdin {

View file

@ -38,6 +38,7 @@ import (
const (
seccompUnconfined = "unconfined"
seccompRuntimeDefault = "runtime/default"
seccompDockerDefault = "docker/default"
seccompLocalhostPrefix = "localhost/"
scopePrefix = "crio"
@ -65,6 +66,11 @@ func addOCIBindMounts(mountLabel string, containerConfig *pb.ContainerConfig, sp
}
}
src, err := resolveSymbolicLink(src)
if err != nil {
return nil, fmt.Errorf("failed to resolve symlink %q: %v", src, err)
}
options := []string{"rw"}
if mount.Readonly {
options = []string{"ro"}
@ -519,12 +525,25 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
specgen.HostSpecific = true
specgen.ClearProcessRlimits()
var readOnlyRootfs bool
var privileged bool
if containerConfig.GetLinux().GetSecurityContext() != nil {
if containerConfig.GetLinux().GetSecurityContext().Privileged {
privileged = true
}
if containerConfig.GetLinux().GetSecurityContext().ReadonlyRootfs {
readOnlyRootfs = true
specgen.SetRootReadonly(true)
}
}
mountLabel := sb.MountLabel()
processLabel := sb.ProcessLabel()
selinuxConfig := containerConfig.GetLinux().GetSecurityContext().GetSelinuxOptions()
if selinuxConfig != nil {
var err error
processLabel, mountLabel, err = getSELinuxLabels(selinuxConfig)
processLabel, mountLabel, err = getSELinuxLabels(selinuxConfig, privileged)
if err != nil {
return nil, err
}
@ -564,19 +583,6 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
}
var readOnlyRootfs bool
var privileged bool
if containerConfig.GetLinux().GetSecurityContext() != nil {
if containerConfig.GetLinux().GetSecurityContext().Privileged {
privileged = true
}
if containerConfig.GetLinux().GetSecurityContext().ReadonlyRootfs {
readOnlyRootfs = true
specgen.SetRootReadonly(true)
}
}
// set this container's apparmor profile if it is set by sandbox
if s.appArmorEnabled && !privileged {
appArmorProfileName := s.getAppArmorProfileName(sb.Annotations(), metadata.Name)
@ -667,6 +673,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
if privileged {
// this is setting correct capabilities as well for privileged mode
specgen.SetupPrivileged(true)
setOCIBindMountsPrivileged(&specgen)
} else {
toCAPPrefixed := func(cap string) string {
if !strings.HasPrefix(strings.ToLower(cap), "cap_") {
@ -714,10 +721,9 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
}
}
specgen.SetProcessSelinuxLabel(processLabel)
}
specgen.SetLinuxMountLabel(sb.MountLabel())
specgen.SetProcessSelinuxLabel(processLabel)
specgen.SetLinuxMountLabel(mountLabel)
if containerConfig.GetLinux().GetSecurityContext() != nil &&
!containerConfig.GetLinux().GetSecurityContext().Privileged {
@ -862,6 +868,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
specgen.AddAnnotation(annotations.Stdin, fmt.Sprintf("%v", containerConfig.Stdin))
specgen.AddAnnotation(annotations.StdinOnce, fmt.Sprintf("%v", containerConfig.StdinOnce))
specgen.AddAnnotation(annotations.Image, image)
specgen.AddAnnotation(annotations.ResolvPath, sb.InfraContainer().CrioAnnotations()[annotations.ResolvPath])
created := time.Now()
specgen.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano))
@ -884,13 +891,13 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
specgen.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON))
metaname := metadata.Name
if !privileged {
if err = s.setupSeccomp(&specgen, containerName, sb.Annotations()); err != nil {
if err = s.setupSeccomp(&specgen, metaname, sb.Annotations()); err != nil {
return nil, err
}
}
metaname := metadata.Name
attempt := metadata.Attempt
containerInfo, err := s.StorageRuntimeServer().CreateContainer(s.ImageContext(),
sb.Name(), sb.ID(),
@ -1000,7 +1007,9 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
return nil, err
}
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
crioAnnotations := specgen.Spec().Annotations
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
if err != nil {
return nil, err
}
@ -1014,9 +1023,9 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
}
func (s *Server) setupSeccomp(specgen *generate.Generator, cname string, sbAnnotations map[string]string) error {
profile, ok := sbAnnotations["security.alpha.kubernetes.io/seccomp/container/"+cname]
profile, ok := sbAnnotations["container.seccomp.security.alpha.kubernetes.io/"+cname]
if !ok {
profile, ok = sbAnnotations["security.alpha.kubernetes.io/seccomp/pod"]
profile, ok = sbAnnotations["seccomp.security.alpha.kubernetes.io/pod"]
if !ok {
// running w/o seccomp, aka unconfined
profile = seccompUnconfined
@ -1033,18 +1042,13 @@ func (s *Server) setupSeccomp(specgen *generate.Generator, cname string, sbAnnot
specgen.Spec().Linux.Seccomp = nil
return nil
}
if profile == seccompRuntimeDefault {
if profile == seccompRuntimeDefault || profile == seccompDockerDefault {
return seccomp.LoadProfileFromStruct(s.seccompProfile, specgen)
}
if !strings.HasPrefix(profile, seccompLocalhostPrefix) {
return fmt.Errorf("unknown seccomp profile option: %q", profile)
}
//file, err := ioutil.ReadFile(filepath.Join(s.seccompProfileRoot, strings.TrimPrefix(profile, seccompLocalhostPrefix)))
//if err != nil {
//return err
//}
// TODO(runcom): setup from provided node's seccomp profile
// can't do this yet, see https://issues.k8s.io/36997
// FIXME: https://github.com/kubernetes/kubernetes/issues/39128
return nil
}
@ -1106,3 +1110,28 @@ func getUserInfo(rootfs string, userName string) (uint32, uint32, []uint32, erro
return uid, gid, additionalGids, nil
}
func setOCIBindMountsPrivileged(g *generate.Generator) {
spec := g.Spec()
// clear readonly for /sys and cgroup
for i, m := range spec.Mounts {
if spec.Mounts[i].Destination == "/sys" && !spec.Root.Readonly {
clearReadOnly(&spec.Mounts[i])
}
if m.Type == "cgroup" {
clearReadOnly(&spec.Mounts[i])
}
}
spec.Linux.ReadonlyPaths = nil
spec.Linux.MaskedPaths = nil
}
func clearReadOnly(m *rspec.Mount) {
var opt []string
for _, o := range m.Options {
if o != "ro" {
opt = append(opt, o)
}
}
m.Options = opt
}

View file

@ -9,32 +9,12 @@ import (
"github.com/go-zoo/bone"
"github.com/kubernetes-incubator/cri-o/libkpod/sandbox"
"github.com/kubernetes-incubator/cri-o/oci"
"github.com/kubernetes-incubator/cri-o/types"
"github.com/sirupsen/logrus"
)
// ContainerInfo stores information about containers
type ContainerInfo struct {
Name string `json:"name"`
Pid int `json:"pid"`
Image string `json:"image"`
CreatedTime int64 `json:"created_time"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
LogPath string `json:"log_path"`
Root string `json:"root"`
Sandbox string `json:"sandbox"`
IP string `json:"ip_address"`
}
// CrioInfo stores information about the crio daemon
type CrioInfo struct {
StorageDriver string `json:"storage_driver"`
StorageRoot string `json:"storage_root"`
CgroupDriver string `json:"cgroup_driver"`
}
func (s *Server) getInfo() CrioInfo {
return CrioInfo{
func (s *Server) getInfo() types.CrioInfo {
return types.CrioInfo{
StorageDriver: s.config.Config.Storage,
StorageRoot: s.config.Config.Root,
CgroupDriver: s.config.Config.CgroupManager,
@ -47,35 +27,36 @@ var (
errSandboxNotFound = errors.New("sandbox for container not found")
)
func (s *Server) getContainerInfo(id string, getContainerFunc func(id string) *oci.Container, getInfraContainerFunc func(id string) *oci.Container, getSandboxFunc func(id string) *sandbox.Sandbox) (ContainerInfo, error) {
func (s *Server) getContainerInfo(id string, getContainerFunc func(id string) *oci.Container, getInfraContainerFunc func(id string) *oci.Container, getSandboxFunc func(id string) *sandbox.Sandbox) (types.ContainerInfo, error) {
ctr := getContainerFunc(id)
if ctr == nil {
ctr = getInfraContainerFunc(id)
if ctr == nil {
return ContainerInfo{}, errCtrNotFound
return types.ContainerInfo{}, errCtrNotFound
}
}
// TODO(mrunalp): should we call UpdateStatus()?
ctrState := ctr.State()
if ctrState == nil {
return ContainerInfo{}, errCtrStateNil
return types.ContainerInfo{}, errCtrStateNil
}
sb := getSandboxFunc(ctr.Sandbox())
if sb == nil {
logrus.Debugf("can't find sandbox %s for container %s", ctr.Sandbox(), id)
return ContainerInfo{}, errSandboxNotFound
return types.ContainerInfo{}, errSandboxNotFound
}
return ContainerInfo{
Name: ctr.Name(),
Pid: ctrState.Pid,
Image: ctr.Image(),
CreatedTime: ctrState.Created.UnixNano(),
Labels: ctr.Labels(),
Annotations: ctr.Annotations(),
Root: ctr.MountPoint(),
LogPath: ctr.LogPath(),
Sandbox: ctr.Sandbox(),
IP: sb.IP(),
return types.ContainerInfo{
Name: ctr.Name(),
Pid: ctrState.Pid,
Image: ctr.Image(),
CreatedTime: ctrState.Created.UnixNano(),
Labels: ctr.Labels(),
Annotations: ctr.Annotations(),
CrioAnnotations: ctr.CrioAnnotations(),
Root: ctr.MountPoint(),
LogPath: ctr.LogPath(),
Sandbox: ctr.Sandbox(),
IP: sb.IP(),
}, nil
}

View file

@ -67,7 +67,7 @@ func TestGetContainerInfo(t *testing.T) {
"io.kubernetes.test1": "value1",
}
getContainerFunc := func(id string) *oci.Container {
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
if err != nil {
t.Fatal(err)
}
@ -181,7 +181,7 @@ func TestGetContainerInfoCtrStateNil(t *testing.T) {
labels := map[string]string{}
annotations := map[string]string{}
getContainerFunc := func(id string) *oci.Container {
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
if err != nil {
t.Fatal(err)
}
@ -212,7 +212,7 @@ func TestGetContainerInfoSandboxNotFound(t *testing.T) {
labels := map[string]string{}
annotations := map[string]string{}
getContainerFunc := func(id string) *oci.Container {
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
if err != nil {
t.Fatal(err)
}

View file

@ -16,7 +16,7 @@ func (s *Server) networkStart(hostNetwork bool, sb *sandbox.Sandbox) (string, er
return s.BindAddress(), nil
}
podNetwork := newPodNetwork(sb.Namespace(), sb.KubeName(), sb.ID(), sb.NetNsPath())
podNetwork := newPodNetwork(sb)
err := s.netPlugin.SetUpPod(podNetwork)
if err != nil {
return "", fmt.Errorf("failed to create pod network sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
@ -59,7 +59,7 @@ func (s *Server) networkStop(hostNetwork bool, sb *sandbox.Sandbox) error {
sb.Name(), sb.ID(), err)
}
podNetwork := newPodNetwork(sb.Namespace(), sb.KubeName(), sb.ID(), sb.NetNsPath())
podNetwork := newPodNetwork(sb)
if err := s.netPlugin.TearDownPod(podNetwork); err != nil {
logrus.Warnf("failed to destroy network for pod sandbox %s(%s): %v",
sb.Name(), sb.ID(), err)

View file

@ -247,16 +247,20 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
return nil, fmt.Errorf("requested logDir for sbox id %s is a relative path: %s", id, logDir)
}
// Don't use SELinux separation with Host Pid or IPC Namespace,
if !req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostPid && !req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostIpc {
processLabel, mountLabel, err = getSELinuxLabels(req.GetConfig().GetLinux().GetSecurityContext().GetSelinuxOptions())
if err != nil {
return nil, err
}
g.SetProcessSelinuxLabel(processLabel)
g.SetLinuxMountLabel(mountLabel)
privileged := s.privilegedSandbox(req)
processLabel, mountLabel, err = getSELinuxLabels(req.GetConfig().GetLinux().GetSecurityContext().GetSelinuxOptions(), privileged)
if err != nil {
return nil, err
}
// Don't use SELinux separation with Host Pid or IPC Namespace or privileged.
if req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostPid || req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostIpc {
processLabel, mountLabel = "", ""
}
g.SetProcessSelinuxLabel(processLabel)
g.SetLinuxMountLabel(mountLabel)
// create shm mount for the pod containers.
var shmPath string
if req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostIpc {
@ -308,7 +312,6 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
}
g.SetHostname(hostname)
privileged := s.privilegedSandbox(req)
trusted := s.trustedSandbox(req)
g.AddAnnotation(annotations.Metadata, string(metadataJSON))
g.AddAnnotation(annotations.Labels, string(labelsJSON))
@ -472,7 +475,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
g.AddAnnotation(annotations.HostnamePath, hostnamePath)
sb.AddHostnamePath(hostnamePath)
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
if err != nil {
return nil, err
}
@ -557,7 +560,10 @@ func (s *Server) setPodSandboxMountLabel(id, mountLabel string) error {
return s.StorageRuntimeServer().SetContainerMetadata(id, storageMetadata)
}
func getSELinuxLabels(selinuxOptions *pb.SELinuxOption) (processLabel string, mountLabel string, err error) {
func getSELinuxLabels(selinuxOptions *pb.SELinuxOption, privileged bool) (processLabel string, mountLabel string, err error) {
if privileged {
return "", "", nil
}
labels := []string{}
if selinuxOptions != nil {
if selinuxOptions.User != "" {

View file

@ -11,6 +11,7 @@ import (
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/generate"
libseccomp "github.com/seccomp/libseccomp-golang"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
@ -24,6 +25,7 @@ func IsEnabled() bool {
enabled = true
}
}
logrus.Debugf("seccomp status: %v", enabled)
return enabled
}

View file

@ -7,6 +7,7 @@ import (
"strings"
"github.com/cri-o/ocicni/pkg/ocicni"
"github.com/kubernetes-incubator/cri-o/libkpod/sandbox"
"github.com/opencontainers/runtime-tools/validate"
"github.com/syndtr/gocapability/capability"
)
@ -149,12 +150,12 @@ func SysctlsFromPodAnnotation(annotation string) ([]Sysctl, error) {
return sysctls, nil
}
func newPodNetwork(namespace, name, id, netns string) ocicni.PodNetwork {
func newPodNetwork(sb *sandbox.Sandbox) ocicni.PodNetwork {
return ocicni.PodNetwork{
Name: name,
Namespace: namespace,
ID: id,
NetNS: netns,
Name: sb.KubeName(),
Namespace: sb.Namespace(),
ID: sb.ID(),
NetNS: sb.NetNsPath(),
}
}

View file

@ -255,6 +255,53 @@ function teardown() {
stop_crio
}
@test "ctr log max" {
LOG_SIZE_MAX_LIMIT=10000 start_crio
run crioctl pod run --config "$TESTDATA"/sandbox_config.json
echo "$output"
[ "$status" -eq 0 ]
pod_id="$output"
run crioctl pod list
echo "$output"
[ "$status" -eq 0 ]
# Create a new container.
newconfig=$(mktemp --tmpdir crio-config.XXXXXX.json)
cp "$TESTDATA"/container_config_logging.json "$newconfig"
sed -i 's|"%shellcommand%"|"for i in $(seq 250); do echo $i; done"|' "$newconfig"
run crioctl ctr create --config "$newconfig" --pod "$pod_id"
echo "$output"
[ "$status" -eq 0 ]
ctr_id="$output"
run crioctl ctr start --id "$ctr_id"
echo "$output"
[ "$status" -eq 0 ]
sleep 6
run crioctl ctr status --id "$ctr_id"
[ "$status" -eq 0 ]
run crioctl ctr remove --id "$ctr_id"
echo "$output"
[ "$status" -eq 0 ]
# Check that the output is what we expect.
logpath="$DEFAULT_LOG_PATH/$pod_id/$ctr_id.log"
[ -f "$logpath" ]
echo "$logpath :: $(cat "$logpath")"
len=$(wc -l "$logpath" | awk '{print $1}')
[ $len -lt 250 ]
run crioctl pod stop --id "$pod_id"
echo "$output"
[ "$status" -eq 0 ]
run crioctl pod remove --id "$pod_id"
echo "$output"
[ "$status" -eq 0 ]
cleanup_ctrs
cleanup_pods
stop_crio
}
# regression test for #127
@test "ctrs status for a pod" {
start_crio

View file

@ -56,6 +56,8 @@ CGROUP_MANAGER=${CGROUP_MANAGER:-cgroupfs}
IMAGE_VOLUMES=${IMAGE_VOLUMES:-mkdir}
# Container pids limit
PIDS_LIMIT=${PIDS_LIMIT:-1024}
# Log size max limit
LOG_SIZE_MAX_LIMIT=${LOG_SIZE_MAX_LIMIT:--1}
TESTDIR=$(mktemp -d)
@ -231,7 +233,7 @@ function start_crio() {
"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTS --runroot "$TESTDIR/crio-run" --image-name=mrunalp/image-volume-test --import-from=dir:"$ARTIFACTS_PATH"/image-volume-test-image --add-name=docker.io/library/mrunalp/image-volume-test --signature-policy="$INTEGRATION_ROOT"/policy.json
"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTS --runroot "$TESTDIR/crio-run" --image-name=busybox:latest --import-from=dir:"$ARTIFACTS_PATH"/busybox-image --add-name=docker.io/library/busybox:latest --signature-policy="$INTEGRATION_ROOT"/policy.json
"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTS --runroot "$TESTDIR/crio-run" --image-name=runcom/stderr-test:latest --import-from=dir:"$ARTIFACTS_PATH"/stderr-test --add-name=docker.io/runcom/stderr-test:latest --signature-policy="$INTEGRATION_ROOT"/policy.json
"$CRIO_BINARY" ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --config /dev/null config >$CRIO_CONFIG
"$CRIO_BINARY" ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --log-size-max "$LOG_SIZE_MAX_LIMIT" --config /dev/null config >$CRIO_CONFIG
# Prepare the CNI configuration files, we're running with non host networking by default
if [[ -n "$4" ]]; then

View file

@ -12,6 +12,11 @@ function teardown() {
echo "$out"
[[ "$out" =~ "\"cgroup_driver\":\"$CGROUP_MANAGER\"" ]]
[[ "$out" =~ "\"storage_root\":\"$TESTDIR/crio\"" ]]
run crioctl info
echo "$output"
[ "$status" -eq 0 ]
[[ "$output" =~ "\"cgroup_driver\": \"$CGROUP_MANAGER\"" ]]
[[ "$output" =~ "\"storage_root\": \"$TESTDIR/crio\"" ]]
stop_crio
}
@ -32,13 +37,20 @@ function teardown() {
[[ "$out" =~ "\"sandbox\":\"$pod_id\"" ]]
[[ "$out" =~ "\"image\":\"redis:alpine\"" ]]
run crioctl ctr inspect --id $ctr_id
echo "$output"
[ "$status" -eq 0 ]
[[ "$output" =~ "\"sandbox\": \"$pod_id\"" ]]
[[ "$output" =~ "\"image\": \"redis:alpine\"" ]]
inet=`crioctl ctr execsync --id $ctr_id ip addr show dev eth0 scope global 2>&1 | grep inet`
IFS=" "
ip=`parse_pod_ip $inet`
[[ "$out" =~ "\"ip_address\":\"$ip\"" ]]
[[ "$out" =~ "\"name\":\"k8s_container1_podsandbox1_redhat.test.crio_redhat-test-crio_1\"" ]]
[[ "$output" =~ "\"ip_address\": \"$ip\"" ]]
[[ "$output" =~ "\"name\": \"k8s_container1_podsandbox1_redhat.test.crio_redhat-test-crio_1\"" ]]
# TODO: add some other check based on the json below:

View file

@ -21,7 +21,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/container\/k8s_testname_seccomp_1_redhat\.test\.crio_redhat-test-crio_0": "unconfined"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp1.json
sed -e 's/%VALUE%/,"container\.seccomp\.security\.alpha\.kubernetes\.io\/testname": "unconfined"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp1.json
run crioctl pod run --name seccomp1 --config "$TESTDIR"/seccomp1.json
echo "$output"
[ "$status" -eq 0 ]
@ -57,7 +57,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/container\/k8s_testname2_seccomp2_redhat\.test\.crio_redhat-test-crio_0": "runtime\/default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp2.json
sed -e 's/%VALUE%/,"container\.seccomp\.security\.alpha\.kubernetes\.io\/testname2": "runtime\/default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp2.json
run crioctl pod run --name seccomp2 --config "$TESTDIR"/seccomp2.json
echo "$output"
[ "$status" -eq 0 ]
@ -94,7 +94,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/container\/k8s_testname3_seccomp3_redhat\.test\.crio_redhat-test-crio_1": "notgood"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp3.json
sed -e 's/%VALUE%/,"container\.seccomp\.security\.alpha\.kubernetes\.io\/testname3": "notgood"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp3.json
run crioctl pod run --name seccomp3 --config "$TESTDIR"/seccomp3.json
echo "$output"
[ "$status" -eq 0 ]
@ -145,7 +145,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/container\/redhat\.test\.crio-seccomp2-1-testname2-0-not-exists": "unconfined", "security\.alpha\.kubernetes\.io\/seccomp\/pod": "runtime\/default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp5.json
sed -e 's/%VALUE%/,"container\.seccomp\.security\.alpha\.kubernetes\.io\/redhat\.test\.crio-seccomp2-1-testname2-0-not-exists": "unconfined", "seccomp\.security\.alpha\.kubernetes\.io\/pod": "runtime\/default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp5.json
run crioctl pod run --name seccomp5 --config "$TESTDIR"/seccomp5.json
echo "$output"
[ "$status" -eq 0 ]
@ -185,7 +185,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/container\/redhat\.test\.crio-seccomp6-1-testname6-0-not-exists": "runtime-default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp6.json
sed -e 's/%VALUE%/,"container\.seccomp\.security\.alpha\.kubernetes\.io\/redhat\.test\.crio-seccomp6-1-testname6-0-not-exists": "runtime-default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp6.json
run crioctl pod run --name seccomp6 --config "$TESTDIR"/seccomp6.json
echo "$output"
[ "$status" -eq 0 ]
@ -221,7 +221,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/pod": "unconfined"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp1.json
sed -e 's/%VALUE%/,"seccomp\.security\.alpha\.kubernetes\.io\/pod": "unconfined"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp1.json
run crioctl pod run --name seccomp1 --config "$TESTDIR"/seccomp1.json
echo "$output"
[ "$status" -eq 0 ]
@ -257,7 +257,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/pod": "runtime\/default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp2.json
sed -e 's/%VALUE%/,"seccomp\.security\.alpha\.kubernetes\.io\/pod": "runtime\/default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp2.json
run crioctl pod run --name seccomp2 --config "$TESTDIR"/seccomp2.json
echo "$output"
[ "$status" -eq 0 ]
@ -295,7 +295,7 @@ function teardown() {
start_crio "$TESTDIR"/seccomp_profile1.json
# 3. test running with pod wrong profile name
sed -e 's/%VALUE%/,"security\.alpha\.kubernetes\.io\/seccomp\/pod": "notgood"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp3.json
sed -e 's/%VALUE%/,"seccomp\.security\.alpha\.kubernetes\.io\/pod": "notgood"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp3.json
run crioctl pod run --name seccomp3 --config "$TESTDIR"/seccomp3.json
echo "$output"
[ "$status" -eq 0 ]
@ -328,3 +328,41 @@ function teardown() {
skip "need https://issues.k8s.io/36997"
}
# test running with ctr docker/default
# test that we cannot run with a syscall blocked by the default seccomp profile
@test "ctr seccomp profiles docker/default" {
# this test requires seccomp, so skip this test if seccomp is not enabled.
enabled=$(is_seccomp_enabled)
if [[ "$enabled" -eq 0 ]]; then
skip "skip this test since seccomp is not enabled."
fi
sed -e 's/"chmod",//' "$CRIO_ROOT"/cri-o/seccomp.json > "$TESTDIR"/seccomp_profile1.json
sed -i 's/"fchmod",//' "$TESTDIR"/seccomp_profile1.json
sed -i 's/"fchmodat",//g' "$TESTDIR"/seccomp_profile1.json
start_crio "$TESTDIR"/seccomp_profile1.json
sed -e 's/%VALUE%/,"container\.seccomp\.security\.alpha\.kubernetes\.io\/testname2": "docker\/default"/g' "$TESTDATA"/sandbox_config_seccomp.json > "$TESTDIR"/seccomp2.json
run crioctl pod run --name seccomp2 --config "$TESTDIR"/seccomp2.json
echo "$output"
[ "$status" -eq 0 ]
pod_id="$output"
run crioctl ctr create --name testname2 --config "$TESTDATA"/container_redis.json --pod "$pod_id"
echo "$output"
[ "$status" -eq 0 ]
ctr_id="$output"
run crioctl ctr start --id "$ctr_id"
echo "$output"
[ "$status" -eq 0 ]
run crioctl ctr execsync --id "$ctr_id" chmod 777 .
echo "$output"
[ "$status" -eq 0 ]
[[ "$output" =~ "Exit code: 1" ]]
[[ "$output" =~ "Operation not permitted" ]]
cleanup_ctrs
cleanup_pods
stop_crio
}

23
types/types.go Normal file
View file

@ -0,0 +1,23 @@
package types
// ContainerInfo stores information about containers
type ContainerInfo struct {
Name string `json:"name"`
Pid int `json:"pid"`
Image string `json:"image"`
CreatedTime int64 `json:"created_time"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
CrioAnnotations map[string]string `json:"crio_annotations"`
LogPath string `json:"log_path"`
Root string `json:"root"`
Sandbox string `json:"sandbox"`
IP string `json:"ip_address"`
}
// CrioInfo stores information about the crio daemon
type CrioInfo struct {
StorageDriver string `json:"storage_driver"`
StorageRoot string `json:"storage_root"`
CgroupDriver string `json:"cgroup_driver"`
}

View file

@ -1,6 +1,5 @@
k8s.io/kubernetes v1.7.5 https://github.com/kubernetes/kubernetes
# https://github.com/kubernetes/client-go#compatibility-matrix
k8s.io/client-go v4.0.0 https://github.com/kubernetes/client-go
k8s.io/kubernetes v1.7.6 https://github.com/kubernetes/kubernetes
k8s.io/client-go release-4.0 https://github.com/kubernetes/client-go
k8s.io/apimachinery release-1.7 https://github.com/kubernetes/apimachinery
k8s.io/apiserver release-1.7 https://github.com/kubernetes/apiserver
#

View file

@ -1138,7 +1138,7 @@ func mergePatchIntoOriginal(original, patch map[string]interface{}, t reflect.Ty
return err
}
case !foundOriginal && !foundPatch:
return nil
continue
}
// Split all items into patch items and server-only items and then enforce the order.

View file

@ -27,26 +27,20 @@ const (
// // alpha: v1.4
// MyFeature() bool
// owner: tallclair
// owner: timstclair
// alpha: v1.5
//
// StreamingProxyRedirects controls whether the apiserver should intercept (and follow)
// redirects from the backend (Kubelet) for streaming requests (exec/attach/port-forward).
StreamingProxyRedirects utilfeature.Feature = "StreamingProxyRedirects"
// owner: tallclair
// owner: timstclair
// alpha: v1.7
//
// AdvancedAuditing enables a much more general API auditing pipeline, which includes support for
// pluggable output backends and an audit policy specifying how different requests should be
// audited.
AdvancedAuditing utilfeature.Feature = "AdvancedAuditing"
// owner: @ilackams
// alpha: v1.7
//
// Enables compression of REST responses (GET and LIST only)
APIResponseCompression utilfeature.Feature = "APIResponseCompression"
)
func init() {
@ -59,5 +53,4 @@ func init() {
var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureSpec{
StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta},
AdvancedAuditing: {Default: false, PreRelease: utilfeature.Alpha},
APIResponseCompression: {Default: false, PreRelease: utilfeature.Alpha},
}

View file

@ -87,10 +87,7 @@ var (
// IsWebSocketRequest returns true if the incoming request contains connection upgrade headers
// for WebSockets.
func IsWebSocketRequest(req *http.Request) bool {
if !strings.EqualFold(req.Header.Get("Upgrade"), "websocket") {
return false
}
return connectionUpgradeRegex.MatchString(strings.ToLower(req.Header.Get("Connection")))
return connectionUpgradeRegex.MatchString(strings.ToLower(req.Header.Get("Connection"))) && strings.ToLower(req.Header.Get("Upgrade")) == "websocket"
}
// IgnoreReceives reads from a WebSocket until it is closed, then returns. If timeout is set, the

View file

@ -183,7 +183,7 @@ func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (r
}
// serverResources returns the supported resources for all groups and versions.
func (d *DiscoveryClient) serverResources(failEarly bool) ([]*metav1.APIResourceList, error) {
func (d *DiscoveryClient) serverResources() ([]*metav1.APIResourceList, error) {
apiGroups, err := d.ServerGroups()
if err != nil {
return nil, err
@ -199,9 +199,6 @@ func (d *DiscoveryClient) serverResources(failEarly bool) ([]*metav1.APIResource
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[gv] = err
if failEarly {
return nil, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
continue
}
@ -245,7 +242,7 @@ func IsGroupDiscoveryFailedError(err error) bool {
}
// serverPreferredResources returns the supported resources with the version preferred by the server.
func (d *DiscoveryClient) serverPreferredResources(failEarly bool) ([]*metav1.APIResourceList, error) {
func (d *DiscoveryClient) serverPreferredResources() ([]*metav1.APIResourceList, error) {
serverGroupList, err := d.ServerGroups()
if err != nil {
return nil, err
@ -265,9 +262,6 @@ func (d *DiscoveryClient) serverPreferredResources(failEarly bool) ([]*metav1.AP
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[groupVersion] = err
if failEarly {
return nil, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
continue
}
@ -312,9 +306,7 @@ func (d *DiscoveryClient) serverPreferredResources(failEarly bool) ([]*metav1.AP
// ServerPreferredResources returns the supported resources with the version preferred by the
// server.
func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
return withRetries(defaultRetries, func(retryEarly bool) ([]*metav1.APIResourceList, error) {
return d.serverPreferredResources(retryEarly)
})
return withRetries(defaultRetries, d.serverPreferredResources)
}
// ServerPreferredNamespacedResources returns the supported namespaced resources with the
@ -391,12 +383,11 @@ func (d *DiscoveryClient) OpenAPISchema() (*spec.Swagger, error) {
}
// withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns.
func withRetries(maxRetries int, f func(failEarly bool) ([]*metav1.APIResourceList, error)) ([]*metav1.APIResourceList, error) {
func withRetries(maxRetries int, f func() ([]*metav1.APIResourceList, error)) ([]*metav1.APIResourceList, error) {
var result []*metav1.APIResourceList
var err error
for i := 0; i < maxRetries; i++ {
failEarly := i < maxRetries-1
result, err = f(failEarly)
result, err = f()
if err == nil {
return result, nil
}

View file

@ -51,7 +51,7 @@ var (
// semantic version is a git hash, but the version itself is no
// longer the direct output of "git describe", but a slight
// translation to be semver compliant.
gitVersion string = "v1.7.3-beta.0+$Format:%h$"
gitVersion string = "v1.7.5-beta.0+$Format:%h$"
gitCommit string = "$Format:%H$" // sha1 from git, output of $(git rev-parse HEAD)
gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"

View file

@ -615,7 +615,7 @@ type EmptyDirVolumeSource struct {
// The default is nil which means that the limit is undefined.
// More info: http://kubernetes.io/docs/user-guide/volumes#emptydir
// +optional
SizeLimit resource.Quantity
SizeLimit *resource.Quantity
}
// StorageMedium defines ways that storage can be allocated to a volume.

File diff suppressed because it is too large Load diff

View file

@ -11488,7 +11488,7 @@ func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) {
_, _, _ = yysep2, yyq2, yy2arr2
const yyr2 bool = false
yyq2[0] = x.Medium != ""
yyq2[1] = true
yyq2[1] = x.SizeLimit != nil
var yynn2 int
if yyr2 || yy2arr2 {
r.EncodeArrayStart(2)
@ -11520,15 +11520,18 @@ func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) {
if yyr2 || yy2arr2 {
z.EncSendContainerState(codecSelfer_containerArrayElem1234)
if yyq2[1] {
yy7 := &x.SizeLimit
yym8 := z.EncBinary()
_ = yym8
if false {
} else if z.HasExtensions() && z.EncExt(yy7) {
} else if !yym8 && z.IsJSONHandle() {
z.EncJSONMarshal(yy7)
if x.SizeLimit == nil {
r.EncodeNil()
} else {
z.EncFallback(yy7)
yym7 := z.EncBinary()
_ = yym7
if false {
} else if z.HasExtensions() && z.EncExt(x.SizeLimit) {
} else if !yym7 && z.IsJSONHandle() {
z.EncJSONMarshal(x.SizeLimit)
} else {
z.EncFallback(x.SizeLimit)
}
}
} else {
r.EncodeNil()
@ -11538,15 +11541,18 @@ func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) {
z.EncSendContainerState(codecSelfer_containerMapKey1234)
r.EncodeString(codecSelferC_UTF81234, string("sizeLimit"))
z.EncSendContainerState(codecSelfer_containerMapValue1234)
yy9 := &x.SizeLimit
yym10 := z.EncBinary()
_ = yym10
if false {
} else if z.HasExtensions() && z.EncExt(yy9) {
} else if !yym10 && z.IsJSONHandle() {
z.EncJSONMarshal(yy9)
if x.SizeLimit == nil {
r.EncodeNil()
} else {
z.EncFallback(yy9)
yym8 := z.EncBinary()
_ = yym8
if false {
} else if z.HasExtensions() && z.EncExt(x.SizeLimit) {
} else if !yym8 && z.IsJSONHandle() {
z.EncJSONMarshal(x.SizeLimit)
} else {
z.EncFallback(x.SizeLimit)
}
}
}
}
@ -11620,17 +11626,21 @@ func (x *EmptyDirVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decode
}
case "sizeLimit":
if r.TryDecodeAsNil() {
x.SizeLimit = pkg3_resource.Quantity{}
if x.SizeLimit != nil {
x.SizeLimit = nil
}
} else {
yyv5 := &x.SizeLimit
if x.SizeLimit == nil {
x.SizeLimit = new(pkg3_resource.Quantity)
}
yym6 := z.DecBinary()
_ = yym6
if false {
} else if z.HasExtensions() && z.DecExt(yyv5) {
} else if z.HasExtensions() && z.DecExt(x.SizeLimit) {
} else if !yym6 && z.IsJSONHandle() {
z.DecJSONUnmarshal(yyv5)
z.DecJSONUnmarshal(x.SizeLimit)
} else {
z.DecFallback(yyv5, false)
z.DecFallback(x.SizeLimit, false)
}
}
default:
@ -11676,17 +11686,21 @@ func (x *EmptyDirVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Deco
}
z.DecSendContainerState(codecSelfer_containerArrayElem1234)
if r.TryDecodeAsNil() {
x.SizeLimit = pkg3_resource.Quantity{}
if x.SizeLimit != nil {
x.SizeLimit = nil
}
} else {
yyv9 := &x.SizeLimit
if x.SizeLimit == nil {
x.SizeLimit = new(pkg3_resource.Quantity)
}
yym10 := z.DecBinary()
_ = yym10
if false {
} else if z.HasExtensions() && z.DecExt(yyv9) {
} else if z.HasExtensions() && z.DecExt(x.SizeLimit) {
} else if !yym10 && z.IsJSONHandle() {
z.DecJSONUnmarshal(yyv9)
z.DecJSONUnmarshal(x.SizeLimit)
} else {
z.DecFallback(yyv9, false)
z.DecFallback(x.SizeLimit, false)
}
}
for {

View file

@ -700,7 +700,7 @@ type EmptyDirVolumeSource struct {
// The default is nil which means that the limit is undefined.
// More info: http://kubernetes.io/docs/user-guide/volumes#emptydir
// +optional
SizeLimit resource.Quantity `json:"sizeLimit,omitempty" protobuf:"bytes,2,opt,name=sizeLimit"`
SizeLimit *resource.Quantity `json:"sizeLimit,omitempty" protobuf:"bytes,2,opt,name=sizeLimit"`
}
// Represents a Glusterfs mount that lasts the lifetime of a pod.

View file

@ -21,6 +21,7 @@ limitations under the License.
package v1
import (
resource "k8s.io/apimachinery/pkg/api/resource"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
conversion "k8s.io/apimachinery/pkg/conversion"
runtime "k8s.io/apimachinery/pkg/runtime"
@ -1240,7 +1241,7 @@ func Convert_api_DownwardAPIVolumeSource_To_v1_DownwardAPIVolumeSource(in *api.D
func autoConvert_v1_EmptyDirVolumeSource_To_api_EmptyDirVolumeSource(in *EmptyDirVolumeSource, out *api.EmptyDirVolumeSource, s conversion.Scope) error {
out.Medium = api.StorageMedium(in.Medium)
out.SizeLimit = in.SizeLimit
out.SizeLimit = (*resource.Quantity)(unsafe.Pointer(in.SizeLimit))
return nil
}
@ -1251,7 +1252,7 @@ func Convert_v1_EmptyDirVolumeSource_To_api_EmptyDirVolumeSource(in *EmptyDirVol
func autoConvert_api_EmptyDirVolumeSource_To_v1_EmptyDirVolumeSource(in *api.EmptyDirVolumeSource, out *EmptyDirVolumeSource, s conversion.Scope) error {
out.Medium = StorageMedium(in.Medium)
out.SizeLimit = in.SizeLimit
out.SizeLimit = (*resource.Quantity)(unsafe.Pointer(in.SizeLimit))
return nil
}

View file

@ -21,6 +21,7 @@ limitations under the License.
package v1
import (
resource "k8s.io/apimachinery/pkg/api/resource"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
conversion "k8s.io/apimachinery/pkg/conversion"
runtime "k8s.io/apimachinery/pkg/runtime"
@ -858,7 +859,11 @@ func DeepCopy_v1_EmptyDirVolumeSource(in interface{}, out interface{}, c *conver
in := in.(*EmptyDirVolumeSource)
out := out.(*EmptyDirVolumeSource)
*out = *in
out.SizeLimit = in.SizeLimit.DeepCopy()
if in.SizeLimit != nil {
in, out := &in.SizeLimit, &out.SizeLimit
*out = new(resource.Quantity)
**out = (*in).DeepCopy()
}
return nil
}
}

View file

@ -399,10 +399,13 @@ func validateVolumeSource(source *api.VolumeSource, fldPath *field.Path) field.E
if source.EmptyDir != nil {
numVolumes++
if !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) {
unsetSizeLimit := resource.Quantity{}
if unsetSizeLimit.Cmp(source.EmptyDir.SizeLimit) != 0 {
if source.EmptyDir.SizeLimit != nil && source.EmptyDir.SizeLimit.Cmp(resource.Quantity{}) != 0 {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("sizeLimit"), "SizeLimit field disabled by feature-gate for EmptyDir volumes"))
}
} else {
if source.EmptyDir.SizeLimit != nil && source.EmptyDir.SizeLimit.Cmp(resource.Quantity{}) < 0 {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("sizeLimit"), "SizeLimit field must be a valid resource quantity"))
}
}
}
if source.HostPath != nil {
@ -3353,6 +3356,16 @@ func ValidateNodeUpdate(node, oldNode *api.Node) field.ErrorList {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "podCIDR"), "node updates may not change podCIDR except from \"\" to valid"))
}
}
// Allow controller manager updating provider ID when not set
if len(oldNode.Spec.ProviderID) == 0 {
oldNode.Spec.ProviderID = node.Spec.ProviderID
} else {
if oldNode.Spec.ProviderID != node.Spec.ProviderID {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "providerID"), "node updates may not change providerID except from \"\" to valid"))
}
}
// TODO: move reset function to its own location
// Ignore metadata changes now that they have been tested
oldNode.ObjectMeta = node.ObjectMeta

View file

@ -21,6 +21,7 @@ limitations under the License.
package api
import (
resource "k8s.io/apimachinery/pkg/api/resource"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
conversion "k8s.io/apimachinery/pkg/conversion"
fields "k8s.io/apimachinery/pkg/fields"
@ -860,7 +861,11 @@ func DeepCopy_api_EmptyDirVolumeSource(in interface{}, out interface{}, c *conve
in := in.(*EmptyDirVolumeSource)
out := out.(*EmptyDirVolumeSource)
*out = *in
out.SizeLimit = in.SizeLimit.DeepCopy()
if in.SizeLimit != nil {
in, out := &in.SizeLimit, &out.SizeLimit
*out = new(resource.Quantity)
**out = (*in).DeepCopy()
}
return nil
}
}

View file

@ -3314,89 +3314,88 @@ func init() {
}
var fileDescriptorGenerated = []byte{
// 1331 bytes of a gzipped FileDescriptorProto
// 1323 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x57, 0x5b, 0x6f, 0x1b, 0x45,
0x1b, 0xce, 0x3a, 0x4e, 0x9a, 0x6f, 0x9c, 0x26, 0xfd, 0xa6, 0x55, 0xeb, 0xa6, 0xd4, 0x8e, 0x56,
0x08, 0xb5, 0x08, 0x76, 0xa9, 0x29, 0x88, 0x0a, 0x01, 0x8a, 0xcd, 0xa1, 0x15, 0x71, 0x0f, 0xd3,
0x50, 0x21, 0x40, 0x82, 0xc9, 0x7a, 0xea, 0x0c, 0xf1, 0x1e, 0xb4, 0x33, 0xb6, 0x48, 0xa5, 0x4a,
0xdc, 0x70, 0x87, 0x04, 0x37, 0xfc, 0x04, 0x24, 0xfe, 0x01, 0xd7, 0x20, 0x21, 0xf5, 0xb2, 0x97,
0xe5, 0xc6, 0xa2, 0xee, 0x1d, 0x3f, 0x21, 0x12, 0x07, 0xcd, 0x61, 0x4f, 0x5e, 0x6f, 0x1a, 0x87,
0xb4, 0x82, 0x3b, 0x7b, 0xe6, 0x7d, 0x9f, 0xe7, 0x3d, 0x3c, 0xf3, 0xce, 0x2c, 0x78, 0x6b, 0xfb,
0x35, 0x66, 0x51, 0xdf, 0xde, 0xee, 0x6f, 0x92, 0xd0, 0x23, 0x9c, 0x30, 0x3b, 0xd8, 0xee, 0xda,
0x38, 0xa0, 0xcc, 0xc6, 0x7d, 0xee, 0x33, 0x07, 0xf7, 0xa8, 0xd7, 0xb5, 0x07, 0x0d, 0xdc, 0x0b,
0xb6, 0xf0, 0x05, 0xbb, 0x4b, 0x3c, 0x12, 0x62, 0x4e, 0x3a, 0x56, 0x10, 0xfa, 0xdc, 0x87, 0xb6,
0x02, 0xb0, 0x12, 0x00, 0x2b, 0xd8, 0xee, 0x5a, 0x02, 0xc0, 0x4a, 0x01, 0x58, 0x11, 0xc0, 0xca,
0x8b, 0x5d, 0xca, 0xb7, 0xfa, 0x9b, 0x96, 0xe3, 0xbb, 0x76, 0xd7, 0xef, 0xfa, 0xb6, 0xc4, 0xd9,
0xec, 0xdf, 0x96, 0xff, 0xe4, 0x1f, 0xf9, 0x4b, 0xe1, 0xaf, 0x5c, 0xd4, 0x01, 0xe2, 0x80, 0xba,
0xd8, 0xd9, 0xa2, 0x1e, 0x09, 0x77, 0xa2, 0x10, 0xed, 0x90, 0x30, 0xbf, 0x1f, 0x3a, 0x64, 0x3c,
0xaa, 0x3d, 0xbd, 0x98, 0xed, 0x12, 0x8e, 0xed, 0x41, 0x2e, 0x97, 0x15, 0xbb, 0xc8, 0x2b, 0xec,
0x7b, 0x9c, 0xba, 0x79, 0x9a, 0x57, 0x1f, 0xe7, 0xc0, 0x9c, 0x2d, 0xe2, 0xe2, 0x9c, 0xdf, 0xcb,
0x45, 0x7e, 0x7d, 0x4e, 0x7b, 0x36, 0xf5, 0x38, 0xe3, 0x61, 0xce, 0xe9, 0x85, 0xc2, 0x56, 0x4d,
0xca, 0xe5, 0xd2, 0x7e, 0x1b, 0x9b, 0x73, 0x35, 0xbf, 0x33, 0xc0, 0x99, 0x56, 0xe8, 0x33, 0x76,
0x8b, 0x84, 0x8c, 0xfa, 0xde, 0xb5, 0xcd, 0xcf, 0x89, 0xc3, 0x11, 0xb9, 0x4d, 0x42, 0xe2, 0x39,
0x04, 0xae, 0x82, 0xf2, 0x36, 0xf5, 0x3a, 0x55, 0x63, 0xd5, 0x38, 0xf7, 0xbf, 0xe6, 0xe2, 0xbd,
0x61, 0x7d, 0x66, 0x34, 0xac, 0x97, 0xdf, 0xa7, 0x5e, 0x07, 0xc9, 0x1d, 0x61, 0xe1, 0x61, 0x97,
0x54, 0x4b, 0x59, 0x8b, 0xab, 0xd8, 0x25, 0x48, 0xee, 0xc0, 0x06, 0x00, 0x38, 0xa0, 0x9a, 0xa0,
0x3a, 0x2b, 0xed, 0xa0, 0xb6, 0x03, 0x6b, 0xd7, 0xaf, 0xe8, 0x1d, 0x94, 0xb2, 0x32, 0x1f, 0x95,
0xc0, 0xa9, 0xcb, 0x7e, 0x48, 0xef, 0xf8, 0x1e, 0xc7, 0xbd, 0xeb, 0x7e, 0x67, 0x4d, 0xe7, 0x41,
0x42, 0xf8, 0x19, 0x58, 0x10, 0x5d, 0xed, 0x60, 0x8e, 0x65, 0x5c, 0x95, 0xc6, 0x4b, 0x96, 0x56,
0x66, 0xba, 0xc8, 0x89, 0x36, 0x85, 0xb5, 0x35, 0xb8, 0x60, 0xa9, 0xe4, 0xda, 0x84, 0xe3, 0x84,
0x3f, 0x59, 0x43, 0x31, 0x2a, 0xf4, 0x40, 0x99, 0x05, 0xc4, 0x91, 0x39, 0x55, 0x1a, 0xeb, 0xd6,
0x94, 0xba, 0xb7, 0x0a, 0x22, 0xbf, 0x19, 0x10, 0x27, 0xa9, 0x90, 0xf8, 0x87, 0x24, 0x0f, 0x1c,
0x80, 0x79, 0xc6, 0x31, 0xef, 0x33, 0x59, 0x9d, 0x4a, 0xe3, 0xea, 0xa1, 0x31, 0x4a, 0xd4, 0xe6,
0x92, 0xe6, 0x9c, 0x57, 0xff, 0x91, 0x66, 0x33, 0xbf, 0x99, 0x05, 0xab, 0x05, 0x9e, 0x2d, 0xdf,
0xeb, 0x50, 0x4e, 0x7d, 0x0f, 0x5e, 0x06, 0x65, 0xbe, 0x13, 0x10, 0x2d, 0x81, 0x8b, 0x51, 0xf8,
0x1b, 0x3b, 0x01, 0xd9, 0x1d, 0xd6, 0x9f, 0x7d, 0x9c, 0xbf, 0xb0, 0x43, 0x12, 0x01, 0xde, 0x8a,
0xd3, 0x54, 0x62, 0x79, 0x33, 0x1b, 0xd6, 0xee, 0xb0, 0xbe, 0xa7, 0xee, 0xad, 0x18, 0x33, 0x9b,
0x06, 0x1c, 0x00, 0xd8, 0xc3, 0x8c, 0x6f, 0x84, 0xd8, 0x63, 0x8a, 0x93, 0xba, 0x44, 0x97, 0xf2,
0xf9, 0xfd, 0x49, 0x43, 0x78, 0x34, 0x57, 0x74, 0x3c, 0x70, 0x3d, 0x87, 0x86, 0x26, 0x30, 0xc0,
0xe7, 0xc0, 0x7c, 0x48, 0x30, 0xf3, 0xbd, 0x6a, 0x59, 0xe6, 0x13, 0x97, 0x19, 0xc9, 0x55, 0xa4,
0x77, 0xe1, 0x79, 0x70, 0xc4, 0x25, 0x8c, 0xe1, 0x2e, 0xa9, 0xce, 0x49, 0xc3, 0x65, 0x6d, 0x78,
0xa4, 0xad, 0x96, 0x51, 0xb4, 0x6f, 0xfe, 0x6e, 0x80, 0x33, 0x05, 0x15, 0x5d, 0xa7, 0x8c, 0xc3,
0x4f, 0x72, 0xda, 0xb7, 0xf6, 0x97, 0xa0, 0xf0, 0x96, 0xca, 0x3f, 0xa6, 0xb9, 0x17, 0xa2, 0x95,
0x94, 0xee, 0x5d, 0x30, 0x47, 0x39, 0x71, 0x45, 0x7f, 0x66, 0xcf, 0x55, 0x1a, 0x97, 0x0f, 0x4b,
0x86, 0xcd, 0xa3, 0x9a, 0x74, 0xee, 0x8a, 0x80, 0x47, 0x8a, 0xc5, 0xfc, 0xb3, 0x54, 0x98, 0xac,
0x38, 0x1c, 0xf0, 0x6b, 0x03, 0x2c, 0xc9, 0xbf, 0x1b, 0x38, 0xec, 0x12, 0x31, 0x95, 0x74, 0xce,
0xd3, 0x9f, 0xc8, 0x3d, 0x66, 0x5c, 0xf3, 0xa4, 0x0e, 0x6e, 0xe9, 0x66, 0x86, 0x0b, 0x8d, 0x71,
0xc3, 0x0b, 0xa0, 0xe2, 0x52, 0x0f, 0x91, 0xa0, 0x47, 0x1d, 0xac, 0x34, 0x3c, 0xd7, 0x5c, 0x1e,
0x0d, 0xeb, 0x95, 0x76, 0xb2, 0x8c, 0xd2, 0x36, 0xf0, 0x15, 0x50, 0x71, 0xf1, 0x17, 0xb1, 0xcb,
0xac, 0x74, 0x39, 0xae, 0xf9, 0x2a, 0xed, 0x64, 0x0b, 0xa5, 0xed, 0xe0, 0x6d, 0x21, 0x18, 0x1e,
0x52, 0x87, 0x55, 0xcb, 0xb2, 0x13, 0xaf, 0x4f, 0x9d, 0x70, 0x5b, 0xfa, 0xcb, 0x89, 0x93, 0x52,
0x9b, 0xc4, 0x44, 0x11, 0xb8, 0xf9, 0x6b, 0x19, 0x9c, 0xdd, 0x73, 0x72, 0xc0, 0x77, 0x01, 0xf4,
0x37, 0x19, 0x09, 0x07, 0xa4, 0xf3, 0x9e, 0xba, 0x3a, 0xc4, 0x0c, 0x17, 0x5d, 0x98, 0x6d, 0x9e,
0x14, 0x47, 0xe5, 0x5a, 0x6e, 0x17, 0x4d, 0xf0, 0x80, 0x0e, 0x38, 0x2a, 0x0e, 0x90, 0xaa, 0x30,
0xd5, 0xd7, 0xc5, 0x74, 0xa7, 0xf3, 0xff, 0xa3, 0x61, 0xfd, 0xe8, 0x7a, 0x1a, 0x04, 0x65, 0x31,
0xe1, 0x1a, 0x58, 0x76, 0xfa, 0x61, 0x48, 0x3c, 0x3e, 0x56, 0xf1, 0x53, 0xba, 0x02, 0xcb, 0xad,
0xec, 0x36, 0x1a, 0xb7, 0x17, 0x10, 0x1d, 0xc2, 0x68, 0x48, 0x3a, 0x31, 0x44, 0x39, 0x0b, 0xf1,
0x76, 0x76, 0x1b, 0x8d, 0xdb, 0xc3, 0xbb, 0x60, 0x49, 0xa3, 0xea, 0x7a, 0x57, 0xe7, 0x64, 0x0f,
0xdf, 0x38, 0x68, 0x0f, 0xd5, 0x0c, 0x8f, 0x55, 0xda, 0xca, 0x80, 0xa3, 0x31, 0x32, 0xf8, 0x95,
0x01, 0x80, 0x13, 0x0d, 0x4a, 0x56, 0x9d, 0x97, 0xdc, 0x37, 0x0e, 0xeb, 0x24, 0xc7, 0x23, 0x38,
0xb9, 0x41, 0xe3, 0x25, 0x86, 0x52, 0xc4, 0xe6, 0x1f, 0x25, 0x00, 0x12, 0x11, 0xc2, 0x8b, 0x99,
0x5b, 0x64, 0x75, 0xec, 0x16, 0x39, 0xa6, 0x2d, 0xe5, 0x0b, 0x2f, 0x75, 0x63, 0x74, 0xc1, 0xbc,
0x2f, 0x4f, 0xab, 0xd6, 0x4b, 0x6b, 0xea, 0x3c, 0xe2, 0xfb, 0x3d, 0x86, 0x6f, 0x02, 0x31, 0xa2,
0xf5, 0x10, 0xd0, 0xf0, 0xf0, 0x53, 0x50, 0x0e, 0xfc, 0x4e, 0x74, 0xff, 0xae, 0x4d, 0x4d, 0x73,
0xdd, 0xef, 0xb0, 0x0c, 0xc9, 0x82, 0xc8, 0x4e, 0xac, 0x22, 0x09, 0x0c, 0x7d, 0xb0, 0x10, 0xbd,
0x60, 0xa5, 0xa2, 0x2a, 0x8d, 0x77, 0xa6, 0x26, 0x41, 0x1a, 0x20, 0x43, 0xb4, 0x28, 0x66, 0x79,
0xb4, 0x83, 0x62, 0x12, 0xf3, 0xaf, 0x12, 0x58, 0x4c, 0x0b, 0xe8, 0xdf, 0xd1, 0x01, 0xa5, 0xe5,
0x27, 0xdc, 0x01, 0x45, 0xf2, 0x14, 0x3a, 0xa0, 0x88, 0x8a, 0x3a, 0xf0, 0x7d, 0x09, 0xc0, 0xbc,
0xfc, 0x20, 0x07, 0xf3, 0x5c, 0xde, 0x29, 0x4f, 0xe4, 0x32, 0x8b, 0xdf, 0x20, 0xfa, 0xde, 0xd2,
0x5c, 0xe2, 0x11, 0xae, 0xa6, 0xfe, 0xd5, 0xe4, 0xb1, 0x1e, 0x1f, 0xe1, 0x76, 0xbc, 0x83, 0x52,
0x56, 0x90, 0x80, 0x8a, 0xf2, 0xbe, 0x85, 0x7b, 0xfd, 0xe8, 0x41, 0xb5, 0xe7, 0x7b, 0xc3, 0x8a,
0x92, 0xb7, 0x6e, 0xf4, 0xb1, 0xc7, 0x29, 0xdf, 0x49, 0x6e, 0xbb, 0x8d, 0x04, 0x0a, 0xa5, 0x71,
0xcd, 0x1f, 0xc6, 0xeb, 0xa4, 0xf4, 0xfa, 0xdf, 0xa9, 0xd3, 0x16, 0x58, 0xd4, 0x43, 0xf8, 0x9f,
0x14, 0xea, 0x84, 0x66, 0x59, 0x6c, 0xa5, 0xb0, 0x50, 0x06, 0xd9, 0xfc, 0xd9, 0x00, 0xc7, 0xc6,
0x47, 0xcd, 0x58, 0xc8, 0xc6, 0xbe, 0x42, 0xbe, 0x03, 0xa0, 0x4a, 0x78, 0x6d, 0x40, 0x42, 0xdc,
0x25, 0x2a, 0xf0, 0xd2, 0x81, 0x02, 0x8f, 0x9f, 0xcd, 0x1b, 0x39, 0x44, 0x34, 0x81, 0xc5, 0xfc,
0x25, 0x9b, 0x84, 0xea, 0xf6, 0x41, 0x92, 0xb8, 0x0b, 0x8e, 0xeb, 0xea, 0x1c, 0x42, 0x16, 0x67,
0x34, 0xd9, 0xf1, 0x56, 0x1e, 0x12, 0x4d, 0xe2, 0x31, 0x7f, 0x2c, 0x81, 0x13, 0x93, 0x46, 0x32,
0x6c, 0xeb, 0x4f, 0x62, 0x95, 0xc5, 0xa5, 0xf4, 0x27, 0xf1, 0xee, 0xb0, 0x7e, 0x7e, 0xcf, 0x6f,
0x9c, 0x08, 0x30, 0xf5, 0xfd, 0xfc, 0x21, 0xa8, 0x66, 0xaa, 0xf8, 0x01, 0xa7, 0x3d, 0x7a, 0x47,
0xbd, 0xc4, 0xd4, 0x23, 0xf4, 0x99, 0xd1, 0xb0, 0x5e, 0xdd, 0x28, 0xb0, 0x41, 0x85, 0xde, 0xe2,
0xc3, 0x69, 0x82, 0x0a, 0x0e, 0x26, 0xdf, 0x93, 0x53, 0x28, 0xe0, 0xa7, 0x7c, 0xe5, 0x94, 0x0a,
0x0e, 0xb9, 0x72, 0x1f, 0x83, 0xd3, 0xd9, 0xc6, 0xe5, 0x4b, 0x77, 0x76, 0x34, 0xac, 0x9f, 0x6e,
0x15, 0x19, 0xa1, 0x62, 0xff, 0x22, 0xf5, 0xcd, 0x3e, 0x1d, 0xf5, 0x35, 0xad, 0x7b, 0x0f, 0x6b,
0x33, 0xf7, 0x1f, 0xd6, 0x66, 0x1e, 0x3c, 0xac, 0xcd, 0x7c, 0x39, 0xaa, 0x19, 0xf7, 0x46, 0x35,
0xe3, 0xfe, 0xa8, 0x66, 0x3c, 0x18, 0xd5, 0x8c, 0xdf, 0x46, 0x35, 0xe3, 0xdb, 0x47, 0xb5, 0x99,
0x8f, 0x16, 0xa2, 0x61, 0xf8, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4c, 0xa9, 0x91, 0xe9, 0xfe,
0x13, 0x00, 0x00,
0x14, 0xce, 0x3a, 0x4e, 0x1a, 0xc6, 0x69, 0x52, 0xa6, 0x55, 0xeb, 0xa6, 0xd4, 0x8e, 0x56, 0x08,
0xb5, 0x08, 0x76, 0xa9, 0x29, 0x08, 0x84, 0x00, 0xc5, 0xe6, 0xd2, 0x8a, 0xb8, 0x97, 0x69, 0xa8,
0x10, 0x20, 0xc1, 0x64, 0x3d, 0x75, 0x86, 0x78, 0x2f, 0xda, 0x19, 0x5b, 0xa4, 0x52, 0x25, 0x5e,
0x78, 0x43, 0x82, 0x17, 0x7e, 0x02, 0x12, 0xff, 0x80, 0x67, 0x90, 0x90, 0xfa, 0xd8, 0xc7, 0xf2,
0x62, 0x51, 0xf7, 0x8d, 0x9f, 0x50, 0x89, 0x8b, 0xe6, 0xb2, 0x37, 0xaf, 0xd7, 0xad, 0x43, 0x5a,
0xc1, 0x9b, 0x3d, 0x73, 0xce, 0xf7, 0x9d, 0xcb, 0x37, 0x67, 0x66, 0xc1, 0xdb, 0xbb, 0xaf, 0x31,
0x8b, 0xfa, 0xf6, 0x6e, 0x7f, 0x9b, 0x84, 0x1e, 0xe1, 0x84, 0xd9, 0xc1, 0x6e, 0xd7, 0xc6, 0x01,
0x65, 0x36, 0xee, 0x73, 0x9f, 0x39, 0xb8, 0x47, 0xbd, 0xae, 0x3d, 0x68, 0xe0, 0x5e, 0xb0, 0x83,
0xcf, 0xd9, 0x5d, 0xe2, 0x91, 0x10, 0x73, 0xd2, 0xb1, 0x82, 0xd0, 0xe7, 0x3e, 0xb4, 0x15, 0x80,
0x95, 0x00, 0x58, 0xc1, 0x6e, 0xd7, 0x12, 0x00, 0x56, 0x0a, 0xc0, 0x8a, 0x00, 0xd6, 0x5e, 0xec,
0x52, 0xbe, 0xd3, 0xdf, 0xb6, 0x1c, 0xdf, 0xb5, 0xbb, 0x7e, 0xd7, 0xb7, 0x25, 0xce, 0x76, 0xff,
0x86, 0xfc, 0x27, 0xff, 0xc8, 0x5f, 0x0a, 0x7f, 0xed, 0xbc, 0x0e, 0x10, 0x07, 0xd4, 0xc5, 0xce,
0x0e, 0xf5, 0x48, 0xb8, 0x17, 0x85, 0x68, 0x87, 0x84, 0xf9, 0xfd, 0xd0, 0x21, 0xe3, 0x51, 0x4d,
0xf5, 0x62, 0xb6, 0x4b, 0x38, 0xb6, 0x07, 0xb9, 0x5c, 0xd6, 0xec, 0x22, 0xaf, 0xb0, 0xef, 0x71,
0xea, 0xe6, 0x69, 0x5e, 0x7d, 0x98, 0x03, 0x73, 0x76, 0x88, 0x8b, 0x73, 0x7e, 0x2f, 0x17, 0xf9,
0xf5, 0x39, 0xed, 0xd9, 0xd4, 0xe3, 0x8c, 0x87, 0x39, 0xa7, 0x17, 0x0a, 0x5b, 0x35, 0x21, 0x17,
0xf3, 0x7b, 0x03, 0x9c, 0x6a, 0x85, 0x3e, 0x63, 0xd7, 0x49, 0xc8, 0xa8, 0xef, 0x5d, 0xde, 0xfe,
0x82, 0x38, 0x1c, 0x91, 0x1b, 0x24, 0x24, 0x9e, 0x43, 0xe0, 0x3a, 0x28, 0xef, 0x52, 0xaf, 0x53,
0x35, 0xd6, 0x8d, 0x33, 0x4f, 0x35, 0x97, 0x6f, 0x0f, 0xeb, 0x73, 0xa3, 0x61, 0xbd, 0xfc, 0x01,
0xf5, 0x3a, 0x48, 0xee, 0x08, 0x0b, 0x0f, 0xbb, 0xa4, 0x5a, 0xca, 0x5a, 0x5c, 0xc2, 0x2e, 0x41,
0x72, 0x07, 0x36, 0x00, 0xc0, 0x01, 0xd5, 0x04, 0xd5, 0x79, 0x69, 0x07, 0xb5, 0x1d, 0xd8, 0xb8,
0x72, 0x51, 0xef, 0xa0, 0x94, 0x95, 0x79, 0xbf, 0x04, 0x4e, 0x5c, 0xf0, 0x43, 0x7a, 0xd3, 0xf7,
0x38, 0xee, 0x5d, 0xf1, 0x3b, 0x1b, 0x5a, 0x24, 0x24, 0x84, 0x9f, 0x83, 0x25, 0xd1, 0x9a, 0x0e,
0xe6, 0x58, 0xc6, 0x55, 0x69, 0xbc, 0x64, 0x69, 0x79, 0xa5, 0x2b, 0x95, 0x08, 0x4c, 0x58, 0x5b,
0x83, 0x73, 0x96, 0x4a, 0xae, 0x4d, 0x38, 0x4e, 0xf8, 0x93, 0x35, 0x14, 0xa3, 0x42, 0x0f, 0x94,
0x59, 0x40, 0x1c, 0x99, 0x53, 0xa5, 0xb1, 0x69, 0xcd, 0x28, 0x5e, 0xab, 0x20, 0xf2, 0x6b, 0x01,
0x71, 0x92, 0x0a, 0x89, 0x7f, 0x48, 0xf2, 0xc0, 0x01, 0x58, 0x64, 0x1c, 0xf3, 0x3e, 0x93, 0xd5,
0xa9, 0x34, 0x2e, 0x1d, 0x18, 0xa3, 0x44, 0x6d, 0xae, 0x68, 0xce, 0x45, 0xf5, 0x1f, 0x69, 0x36,
0xf3, 0xdb, 0x79, 0xb0, 0x5e, 0xe0, 0xd9, 0xf2, 0xbd, 0x0e, 0xe5, 0xd4, 0xf7, 0xe0, 0x05, 0x50,
0xe6, 0x7b, 0x01, 0xd1, 0x12, 0x38, 0x1f, 0x85, 0xbf, 0xb5, 0x17, 0x90, 0x07, 0xc3, 0xfa, 0xb3,
0x0f, 0xf3, 0x17, 0x76, 0x48, 0x22, 0xc0, 0xeb, 0x71, 0x9a, 0x4a, 0x2c, 0x6f, 0x65, 0xc3, 0x7a,
0x30, 0xac, 0x4f, 0x15, 0xaf, 0x15, 0x63, 0x66, 0xd3, 0x80, 0x03, 0x00, 0x7b, 0x98, 0xf1, 0xad,
0x10, 0x7b, 0x4c, 0x71, 0x52, 0x97, 0xe8, 0x52, 0x3e, 0xff, 0x68, 0xd2, 0x10, 0x1e, 0xcd, 0x35,
0x1d, 0x0f, 0xdc, 0xcc, 0xa1, 0xa1, 0x09, 0x0c, 0xf0, 0x39, 0xb0, 0x18, 0x12, 0xcc, 0x7c, 0xaf,
0x5a, 0x96, 0xf9, 0xc4, 0x65, 0x46, 0x72, 0x15, 0xe9, 0x5d, 0x78, 0x16, 0x1c, 0x72, 0x09, 0x63,
0xb8, 0x4b, 0xaa, 0x0b, 0xd2, 0x70, 0x55, 0x1b, 0x1e, 0x6a, 0xab, 0x65, 0x14, 0xed, 0x9b, 0x7f,
0x18, 0xe0, 0x54, 0x41, 0x45, 0x37, 0x29, 0xe3, 0xf0, 0xd3, 0x9c, 0xf6, 0xad, 0x47, 0x4b, 0x50,
0x78, 0x4b, 0xe5, 0x1f, 0xd1, 0xdc, 0x4b, 0xd1, 0x4a, 0x4a, 0xf7, 0x2e, 0x58, 0xa0, 0x9c, 0xb8,
0xa2, 0x3f, 0xf3, 0x67, 0x2a, 0x8d, 0x0b, 0x07, 0x25, 0xc3, 0xe6, 0x61, 0x4d, 0xba, 0x70, 0x51,
0xc0, 0x23, 0xc5, 0x62, 0xfe, 0x55, 0x2a, 0x4c, 0x56, 0x1c, 0x0e, 0xf8, 0x8d, 0x01, 0x56, 0xe4,
0xdf, 0x2d, 0x1c, 0x76, 0x89, 0x98, 0x4a, 0x3a, 0xe7, 0xd9, 0x4f, 0xe4, 0x94, 0x19, 0xd7, 0x3c,
0xae, 0x83, 0x5b, 0xb9, 0x96, 0xe1, 0x42, 0x63, 0xdc, 0xf0, 0x1c, 0xa8, 0xb8, 0xd4, 0x43, 0x24,
0xe8, 0x51, 0x07, 0x2b, 0x0d, 0x2f, 0x34, 0x57, 0x47, 0xc3, 0x7a, 0xa5, 0x9d, 0x2c, 0xa3, 0xb4,
0x0d, 0x7c, 0x05, 0x54, 0x5c, 0xfc, 0x65, 0xec, 0x32, 0x2f, 0x5d, 0x8e, 0x6a, 0xbe, 0x4a, 0x3b,
0xd9, 0x42, 0x69, 0x3b, 0x78, 0x43, 0x08, 0x86, 0x87, 0xd4, 0x61, 0xd5, 0xb2, 0xec, 0xc4, 0x1b,
0x33, 0x27, 0xdc, 0x96, 0xfe, 0x72, 0xe2, 0xa4, 0xd4, 0x26, 0x31, 0x51, 0x04, 0x6e, 0xfe, 0x56,
0x06, 0xa7, 0xa7, 0x4e, 0x0e, 0xf8, 0x1e, 0x80, 0xfe, 0x36, 0x23, 0xe1, 0x80, 0x74, 0xde, 0x57,
0x57, 0x87, 0x98, 0xe1, 0xa2, 0x0b, 0xf3, 0xcd, 0xe3, 0xe2, 0xa8, 0x5c, 0xce, 0xed, 0xa2, 0x09,
0x1e, 0xd0, 0x01, 0x87, 0xc5, 0x01, 0x52, 0x15, 0xa6, 0xfa, 0xba, 0x98, 0xed, 0x74, 0x3e, 0x3d,
0x1a, 0xd6, 0x0f, 0x6f, 0xa6, 0x41, 0x50, 0x16, 0x13, 0x6e, 0x80, 0x55, 0xa7, 0x1f, 0x86, 0xc4,
0xe3, 0x63, 0x15, 0x3f, 0xa1, 0x2b, 0xb0, 0xda, 0xca, 0x6e, 0xa3, 0x71, 0x7b, 0x01, 0xd1, 0x21,
0x8c, 0x86, 0xa4, 0x13, 0x43, 0x94, 0xb3, 0x10, 0xef, 0x64, 0xb7, 0xd1, 0xb8, 0x3d, 0xbc, 0x05,
0x56, 0x34, 0xaa, 0xae, 0x77, 0x75, 0x41, 0xf6, 0xf0, 0xcd, 0xfd, 0xf6, 0x50, 0xcd, 0xf0, 0x58,
0xa5, 0xad, 0x0c, 0x38, 0x1a, 0x23, 0x83, 0x5f, 0x1b, 0x00, 0x38, 0xd1, 0xa0, 0x64, 0xd5, 0x45,
0xc9, 0x7d, 0xf5, 0xa0, 0x4e, 0x72, 0x3c, 0x82, 0x93, 0x1b, 0x34, 0x5e, 0x62, 0x28, 0x45, 0x6c,
0xfe, 0x59, 0x02, 0x20, 0x11, 0x21, 0x3c, 0x9f, 0xb9, 0x45, 0xd6, 0xc7, 0x6e, 0x91, 0x23, 0xda,
0x52, 0x3e, 0xd3, 0x52, 0x37, 0x46, 0x17, 0x2c, 0xfa, 0xf2, 0xb4, 0x6a, 0xbd, 0xb4, 0x66, 0xce,
0x23, 0xbe, 0xdf, 0x63, 0xf8, 0x26, 0x10, 0x23, 0x5a, 0x0f, 0x01, 0x0d, 0x0f, 0x3f, 0x03, 0xe5,
0xc0, 0xef, 0x44, 0xf7, 0xef, 0xc6, 0xcc, 0x34, 0x57, 0xfc, 0x0e, 0xcb, 0x90, 0x2c, 0x89, 0xec,
0xc4, 0x2a, 0x92, 0xc0, 0xd0, 0x07, 0x4b, 0xd1, 0x33, 0x54, 0x2a, 0xaa, 0xd2, 0x78, 0x77, 0x66,
0x12, 0xa4, 0x01, 0x32, 0x44, 0xcb, 0x62, 0x96, 0x47, 0x3b, 0x28, 0x26, 0x31, 0xff, 0x2e, 0x81,
0xe5, 0xb4, 0x80, 0xfe, 0x1b, 0x1d, 0x50, 0x5a, 0x7e, 0xcc, 0x1d, 0x50, 0x24, 0x4f, 0xa0, 0x03,
0x8a, 0xa8, 0xa8, 0x03, 0x3f, 0x94, 0x00, 0xcc, 0xcb, 0x0f, 0x72, 0xb0, 0xc8, 0xe5, 0x9d, 0xf2,
0x58, 0x2e, 0xb3, 0xf8, 0x0d, 0xa2, 0xef, 0x2d, 0xcd, 0x25, 0x1e, 0xe1, 0x6a, 0xea, 0x5f, 0x4a,
0x1e, 0xeb, 0xf1, 0x11, 0x6e, 0xc7, 0x3b, 0x28, 0x65, 0x05, 0x09, 0xa8, 0x28, 0xef, 0xeb, 0xb8,
0xd7, 0x8f, 0x1e, 0x54, 0x53, 0xdf, 0x1b, 0x56, 0x94, 0xbc, 0x75, 0xb5, 0x8f, 0x3d, 0x4e, 0xf9,
0x5e, 0x72, 0xdb, 0x6d, 0x25, 0x50, 0x28, 0x8d, 0x6b, 0xfe, 0x38, 0x5e, 0x27, 0xa5, 0xd7, 0xff,
0x4f, 0x9d, 0x76, 0xc0, 0xb2, 0x1e, 0xc2, 0xff, 0xa6, 0x50, 0xc7, 0x34, 0xcb, 0x72, 0x2b, 0x85,
0x85, 0x32, 0xc8, 0xe6, 0x2f, 0x06, 0x38, 0x32, 0x3e, 0x6a, 0xc6, 0x42, 0x36, 0x1e, 0x29, 0xe4,
0x9b, 0x00, 0xaa, 0x84, 0x37, 0x06, 0x24, 0xc4, 0x5d, 0xa2, 0x02, 0x2f, 0xed, 0x2b, 0xf0, 0xf8,
0xd9, 0xbc, 0x95, 0x43, 0x44, 0x13, 0x58, 0xcc, 0x5f, 0xb3, 0x49, 0xa8, 0x6e, 0xef, 0x27, 0x89,
0x5b, 0xe0, 0xa8, 0xae, 0xce, 0x01, 0x64, 0x71, 0x4a, 0x93, 0x1d, 0x6d, 0xe5, 0x21, 0xd1, 0x24,
0x1e, 0xf3, 0xa7, 0x12, 0x38, 0x36, 0x69, 0x24, 0xc3, 0xb6, 0xfe, 0x24, 0x56, 0x59, 0xbc, 0x9e,
0xfe, 0x24, 0x7e, 0x30, 0xac, 0x9f, 0x9d, 0xfa, 0x8d, 0x13, 0x01, 0xa6, 0xbe, 0x9f, 0x3f, 0x02,
0xd5, 0x4c, 0x15, 0x3f, 0xe4, 0xb4, 0x47, 0x6f, 0xaa, 0x97, 0x98, 0x7a, 0x84, 0x3e, 0x33, 0x1a,
0xd6, 0xab, 0x5b, 0x05, 0x36, 0xa8, 0xd0, 0x5b, 0x7c, 0x38, 0x4d, 0x50, 0xc1, 0xfe, 0xe4, 0x7b,
0x7c, 0x06, 0x05, 0xfc, 0x9c, 0xaf, 0x9c, 0x52, 0xc1, 0x01, 0x57, 0xee, 0x13, 0x70, 0x32, 0xdb,
0xb8, 0x7c, 0xe9, 0x4e, 0x8f, 0x86, 0xf5, 0x93, 0xad, 0x22, 0x23, 0x54, 0xec, 0x5f, 0xa4, 0xbe,
0xf9, 0x27, 0xa3, 0xbe, 0xa6, 0x75, 0xfb, 0x5e, 0x6d, 0xee, 0xce, 0xbd, 0xda, 0xdc, 0xdd, 0x7b,
0xb5, 0xb9, 0xaf, 0x46, 0x35, 0xe3, 0xf6, 0xa8, 0x66, 0xdc, 0x19, 0xd5, 0x8c, 0xbb, 0xa3, 0x9a,
0xf1, 0xfb, 0xa8, 0x66, 0x7c, 0x77, 0xbf, 0x36, 0xf7, 0xf1, 0x52, 0x34, 0x0c, 0xff, 0x09, 0x00,
0x00, 0xff, 0xff, 0x88, 0x5a, 0x1f, 0xc3, 0xc3, 0x13, 0x00, 0x00,
}

View file

@ -27,7 +27,6 @@ import "k8s.io/apimachinery/pkg/runtime/generated.proto";
import "k8s.io/apimachinery/pkg/runtime/schema/generated.proto";
import "k8s.io/apimachinery/pkg/util/intstr/generated.proto";
import "k8s.io/kubernetes/pkg/api/v1/generated.proto";
import "k8s.io/kubernetes/pkg/apis/autoscaling/v1/generated.proto";
// Package-wide variables from generator "generated".
option go_package = "v2alpha1";