/* Copyright 2014 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package testing import ( "fmt" "net" "os" "os/exec" "path" "strings" "sync" "testing" "time" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" utiltesting "k8s.io/kubernetes/pkg/util/testing" "k8s.io/kubernetes/pkg/util/uuid" . "k8s.io/kubernetes/pkg/volume" ) // fakeVolumeHost is useful for testing volume plugins. type fakeVolumeHost struct { rootDir string kubeClient clientset.Interface pluginMgr VolumePluginMgr cloud cloudprovider.Interface mounter mount.Interface writer io.Writer } func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost { host := &fakeVolumeHost{rootDir: rootDir, kubeClient: kubeClient, cloud: nil} host.mounter = &mount.FakeMounter{} host.writer = &io.StdWriter{} host.pluginMgr.InitPlugins(plugins, host) return host } func (f *fakeVolumeHost) GetPluginDir(podUID string) string { return path.Join(f.rootDir, "plugins", podUID) } func (f *fakeVolumeHost) GetPodVolumeDir(podUID types.UID, pluginName, volumeName string) string { return path.Join(f.rootDir, "pods", string(podUID), "volumes", pluginName, volumeName) } func (f *fakeVolumeHost) GetPodPluginDir(podUID types.UID, pluginName string) string { return path.Join(f.rootDir, "pods", string(podUID), "plugins", pluginName) } func (f *fakeVolumeHost) GetKubeClient() clientset.Interface { return f.kubeClient } func (f *fakeVolumeHost) GetCloudProvider() cloudprovider.Interface { return f.cloud } func (f *fakeVolumeHost) GetMounter() mount.Interface { return f.mounter } func (f *fakeVolumeHost) GetWriter() io.Writer { return f.writer } func (f *fakeVolumeHost) NewWrapperMounter(volName string, spec Spec, pod *v1.Pod, opts VolumeOptions) (Mounter, error) { // The name of wrapper volume is set to "wrapped_{wrapped_volume_name}" wrapperVolumeName := "wrapped_" + volName if spec.Volume != nil { spec.Volume.Name = wrapperVolumeName } plug, err := f.pluginMgr.FindPluginBySpec(&spec) if err != nil { return nil, err } return plug.NewMounter(&spec, pod, opts) } func (f *fakeVolumeHost) NewWrapperUnmounter(volName string, spec Spec, podUID types.UID) (Unmounter, error) { // The name of wrapper volume is set to "wrapped_{wrapped_volume_name}" wrapperVolumeName := "wrapped_" + volName if spec.Volume != nil { spec.Volume.Name = wrapperVolumeName } plug, err := f.pluginMgr.FindPluginBySpec(&spec) if err != nil { return nil, err } return plug.NewUnmounter(spec.Name(), podUID) } // Returns the hostname of the host kubelet is running on func (f *fakeVolumeHost) GetHostName() string { return "fakeHostName" } // Returns host IP or nil in the case of error. func (f *fakeVolumeHost) GetHostIP() (net.IP, error) { return nil, fmt.Errorf("GetHostIP() not implemented") } func (f *fakeVolumeHost) GetNodeAllocatable() (v1.ResourceList, error) { return v1.ResourceList{}, nil } func ProbeVolumePlugins(config VolumeConfig) []VolumePlugin { if _, ok := config.OtherAttributes["fake-property"]; ok { return []VolumePlugin{ &FakeVolumePlugin{ PluginName: "fake-plugin", Host: nil, // SomeFakeProperty: config.OtherAttributes["fake-property"] -- string, may require parsing by plugin }, } } return []VolumePlugin{&FakeVolumePlugin{PluginName: "fake-plugin"}} } // FakeVolumePlugin is useful for testing. It tries to be a fully compliant // plugin, but all it does is make empty directories. // Use as: // volume.RegisterPlugin(&FakePlugin{"fake-name"}) type FakeVolumePlugin struct { sync.RWMutex PluginName string Host VolumeHost Config VolumeConfig LastProvisionerOptions VolumeOptions NewAttacherCallCount int NewDetacherCallCount int Mounters []*FakeVolume Unmounters []*FakeVolume Attachers []*FakeVolume Detachers []*FakeVolume } var _ VolumePlugin = &FakeVolumePlugin{} var _ RecyclableVolumePlugin = &FakeVolumePlugin{} var _ DeletableVolumePlugin = &FakeVolumePlugin{} var _ ProvisionableVolumePlugin = &FakeVolumePlugin{} var _ AttachableVolumePlugin = &FakeVolumePlugin{} func (plugin *FakeVolumePlugin) getFakeVolume(list *[]*FakeVolume) *FakeVolume { volume := &FakeVolume{} *list = append(*list, volume) return volume } func (plugin *FakeVolumePlugin) Init(host VolumeHost) error { plugin.Lock() defer plugin.Unlock() plugin.Host = host return nil } func (plugin *FakeVolumePlugin) GetPluginName() string { plugin.RLock() defer plugin.RUnlock() return plugin.PluginName } func (plugin *FakeVolumePlugin) GetVolumeName(spec *Spec) (string, error) { return spec.Name(), nil } func (plugin *FakeVolumePlugin) CanSupport(spec *Spec) bool { // TODO: maybe pattern-match on spec.Name() to decide? return true } func (plugin *FakeVolumePlugin) RequiresRemount() bool { return false } func (plugin *FakeVolumePlugin) NewMounter(spec *Spec, pod *v1.Pod, opts VolumeOptions) (Mounter, error) { plugin.Lock() defer plugin.Unlock() volume := plugin.getFakeVolume(&plugin.Mounters) volume.PodUID = pod.UID volume.VolName = spec.Name() volume.Plugin = plugin volume.MetricsNil = MetricsNil{} return volume, nil } func (plugin *FakeVolumePlugin) GetMounters() (Mounters []*FakeVolume) { plugin.RLock() defer plugin.RUnlock() return plugin.Mounters } func (plugin *FakeVolumePlugin) NewUnmounter(volName string, podUID types.UID) (Unmounter, error) { plugin.Lock() defer plugin.Unlock() volume := plugin.getFakeVolume(&plugin.Unmounters) volume.PodUID = podUID volume.VolName = volName volume.Plugin = plugin volume.MetricsNil = MetricsNil{} return volume, nil } func (plugin *FakeVolumePlugin) GetUnmounters() (Unmounters []*FakeVolume) { plugin.RLock() defer plugin.RUnlock() return plugin.Unmounters } func (plugin *FakeVolumePlugin) NewAttacher() (Attacher, error) { plugin.Lock() defer plugin.Unlock() plugin.NewAttacherCallCount = plugin.NewAttacherCallCount + 1 return plugin.getFakeVolume(&plugin.Attachers), nil } func (plugin *FakeVolumePlugin) GetAttachers() (Attachers []*FakeVolume) { plugin.RLock() defer plugin.RUnlock() return plugin.Attachers } func (plugin *FakeVolumePlugin) GetNewAttacherCallCount() int { plugin.RLock() defer plugin.RUnlock() return plugin.NewAttacherCallCount } func (plugin *FakeVolumePlugin) NewDetacher() (Detacher, error) { plugin.Lock() defer plugin.Unlock() plugin.NewDetacherCallCount = plugin.NewDetacherCallCount + 1 return plugin.getFakeVolume(&plugin.Detachers), nil } func (plugin *FakeVolumePlugin) GetDetachers() (Detachers []*FakeVolume) { plugin.RLock() defer plugin.RUnlock() return plugin.Detachers } func (plugin *FakeVolumePlugin) GetNewDetacherCallCount() int { plugin.RLock() defer plugin.RUnlock() return plugin.NewDetacherCallCount } func (plugin *FakeVolumePlugin) NewRecycler(pvName string, spec *Spec, eventRecorder RecycleEventRecorder) (Recycler, error) { return &fakeRecycler{"/attributesTransferredFromSpec", MetricsNil{}}, nil } func (plugin *FakeVolumePlugin) NewDeleter(spec *Spec) (Deleter, error) { return &FakeDeleter{"/attributesTransferredFromSpec", MetricsNil{}}, nil } func (plugin *FakeVolumePlugin) NewProvisioner(options VolumeOptions) (Provisioner, error) { plugin.Lock() defer plugin.Unlock() plugin.LastProvisionerOptions = options return &FakeProvisioner{options, plugin.Host}, nil } func (plugin *FakeVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{} } func (plugin *FakeVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error) { return nil, nil } func (plugin *FakeVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { return []string{}, nil } type FakeVolume struct { sync.RWMutex PodUID types.UID VolName string Plugin *FakeVolumePlugin MetricsNil SetUpCallCount int TearDownCallCount int AttachCallCount int DetachCallCount int WaitForAttachCallCount int MountDeviceCallCount int UnmountDeviceCallCount int GetDeviceMountPathCallCount int } func (_ *FakeVolume) GetAttributes() Attributes { return Attributes{ ReadOnly: false, Managed: true, SupportsSELinux: true, } } func (fv *FakeVolume) CanMount() error { return nil } func (fv *FakeVolume) SetUp(fsGroup *int64) error { fv.Lock() defer fv.Unlock() fv.SetUpCallCount++ return fv.SetUpAt(fv.getPath(), fsGroup) } func (fv *FakeVolume) GetSetUpCallCount() int { fv.RLock() defer fv.RUnlock() return fv.SetUpCallCount } func (fv *FakeVolume) SetUpAt(dir string, fsGroup *int64) error { return os.MkdirAll(dir, 0750) } func (fv *FakeVolume) GetPath() string { fv.RLock() defer fv.RUnlock() return fv.getPath() } func (fv *FakeVolume) getPath() string { return path.Join(fv.Plugin.Host.GetPodVolumeDir(fv.PodUID, utilstrings.EscapeQualifiedNameForDisk(fv.Plugin.PluginName), fv.VolName)) } func (fv *FakeVolume) TearDown() error { fv.Lock() defer fv.Unlock() fv.TearDownCallCount++ return fv.TearDownAt(fv.getPath()) } func (fv *FakeVolume) GetTearDownCallCount() int { fv.RLock() defer fv.RUnlock() return fv.TearDownCallCount } func (fv *FakeVolume) TearDownAt(dir string) error { return os.RemoveAll(dir) } func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error) { fv.Lock() defer fv.Unlock() fv.AttachCallCount++ return "", nil } func (fv *FakeVolume) GetAttachCallCount() int { fv.RLock() defer fv.RUnlock() return fv.AttachCallCount } func (fv *FakeVolume) WaitForAttach(spec *Spec, devicePath string, spectimeout time.Duration) (string, error) { fv.Lock() defer fv.Unlock() fv.WaitForAttachCallCount++ return "", nil } func (fv *FakeVolume) GetWaitForAttachCallCount() int { fv.RLock() defer fv.RUnlock() return fv.WaitForAttachCallCount } func (fv *FakeVolume) GetDeviceMountPath(spec *Spec) (string, error) { fv.Lock() defer fv.Unlock() fv.GetDeviceMountPathCallCount++ return "", nil } func (fv *FakeVolume) MountDevice(spec *Spec, devicePath string, deviceMountPath string) error { fv.Lock() defer fv.Unlock() fv.MountDeviceCallCount++ return nil } func (fv *FakeVolume) GetMountDeviceCallCount() int { fv.RLock() defer fv.RUnlock() return fv.MountDeviceCallCount } func (fv *FakeVolume) Detach(deviceMountPath string, nodeName types.NodeName) error { fv.Lock() defer fv.Unlock() fv.DetachCallCount++ return nil } func (fv *FakeVolume) VolumesAreAttached(spec []*Spec, nodeName types.NodeName) (map[*Spec]bool, error) { fv.Lock() defer fv.Unlock() return nil, nil } func (fv *FakeVolume) GetDetachCallCount() int { fv.RLock() defer fv.RUnlock() return fv.DetachCallCount } func (fv *FakeVolume) UnmountDevice(globalMountPath string) error { fv.Lock() defer fv.Unlock() fv.UnmountDeviceCallCount++ return nil } type fakeRecycler struct { path string MetricsNil } func (fr *fakeRecycler) Recycle() error { // nil is success, else error return nil } func (fr *fakeRecycler) GetPath() string { return fr.path } type FakeDeleter struct { path string MetricsNil } func (fd *FakeDeleter) Delete() error { // nil is success, else error return nil } func (fd *FakeDeleter) GetPath() string { return fd.path } type FakeProvisioner struct { Options VolumeOptions Host VolumeHost } func (fc *FakeProvisioner) Provision() (*v1.PersistentVolume, error) { fullpath := fmt.Sprintf("/tmp/hostpath_pv/%s", uuid.NewUUID()) pv := &v1.PersistentVolume{ ObjectMeta: v1.ObjectMeta{ Name: fc.Options.PVName, Annotations: map[string]string{ "kubernetes.io/createdby": "fakeplugin-provisioner", }, }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: fc.Options.PersistentVolumeReclaimPolicy, AccessModes: fc.Options.PVC.Spec.AccessModes, Capacity: v1.ResourceList{ v1.ResourceName(v1.ResourceStorage): fc.Options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)], }, PersistentVolumeSource: v1.PersistentVolumeSource{ HostPath: &v1.HostPathVolumeSource{ Path: fullpath, }, }, }, } return pv, nil } // FindEmptyDirectoryUsageOnTmpfs finds the expected usage of an empty directory existing on // a tmpfs filesystem on this system. func FindEmptyDirectoryUsageOnTmpfs() (*resource.Quantity, error) { tmpDir, err := utiltesting.MkTmpdir("metrics_du_test") if err != nil { return nil, err } out, err := exec.Command("nice", "-n", "19", "du", "-s", "-B", "1", tmpDir).CombinedOutput() if err != nil { return nil, fmt.Errorf("failed command 'du' on %s with error %v", tmpDir, err) } used, err := resource.ParseQuantity(strings.Fields(string(out))[0]) if err != nil { return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err) } used.Format = resource.BinarySI return &used, nil } // VerifyAttachCallCount ensures that at least one of the Attachers for this // plugin has the expectedAttachCallCount number of calls. Otherwise it returns // an error. func VerifyAttachCallCount( expectedAttachCallCount int, fakeVolumePlugin *FakeVolumePlugin) error { for _, attacher := range fakeVolumePlugin.GetAttachers() { actualCallCount := attacher.GetAttachCallCount() if actualCallCount == expectedAttachCallCount { return nil } } return fmt.Errorf( "No attachers have expected AttachCallCount. Expected: <%v>.", expectedAttachCallCount) } // VerifyZeroAttachCalls ensures that all of the Attachers for this plugin have // a zero AttachCallCount. Otherwise it returns an error. func VerifyZeroAttachCalls(fakeVolumePlugin *FakeVolumePlugin) error { for _, attacher := range fakeVolumePlugin.GetAttachers() { actualCallCount := attacher.GetAttachCallCount() if actualCallCount != 0 { return fmt.Errorf( "At least one attacher has non-zero AttachCallCount: <%v>.", actualCallCount) } } return nil } // VerifyWaitForAttachCallCount ensures that at least one of the Mounters for // this plugin has the expectedWaitForAttachCallCount number of calls. Otherwise // it returns an error. func VerifyWaitForAttachCallCount( expectedWaitForAttachCallCount int, fakeVolumePlugin *FakeVolumePlugin) error { for _, attacher := range fakeVolumePlugin.GetAttachers() { actualCallCount := attacher.GetWaitForAttachCallCount() if actualCallCount == expectedWaitForAttachCallCount { return nil } } return fmt.Errorf( "No Attachers have expected WaitForAttachCallCount. Expected: <%v>.", expectedWaitForAttachCallCount) } // VerifyZeroWaitForAttachCallCount ensures that all Attachers for this plugin // have a zero WaitForAttachCallCount. Otherwise it returns an error. func VerifyZeroWaitForAttachCallCount(fakeVolumePlugin *FakeVolumePlugin) error { for _, attacher := range fakeVolumePlugin.GetAttachers() { actualCallCount := attacher.GetWaitForAttachCallCount() if actualCallCount != 0 { return fmt.Errorf( "At least one attacher has non-zero WaitForAttachCallCount: <%v>.", actualCallCount) } } return nil } // VerifyMountDeviceCallCount ensures that at least one of the Mounters for // this plugin has the expectedMountDeviceCallCount number of calls. Otherwise // it returns an error. func VerifyMountDeviceCallCount( expectedMountDeviceCallCount int, fakeVolumePlugin *FakeVolumePlugin) error { for _, attacher := range fakeVolumePlugin.GetAttachers() { actualCallCount := attacher.GetMountDeviceCallCount() if actualCallCount == expectedMountDeviceCallCount { return nil } } return fmt.Errorf( "No Attachers have expected MountDeviceCallCount. Expected: <%v>.", expectedMountDeviceCallCount) } // VerifyZeroMountDeviceCallCount ensures that all Attachers for this plugin // have a zero MountDeviceCallCount. Otherwise it returns an error. func VerifyZeroMountDeviceCallCount(fakeVolumePlugin *FakeVolumePlugin) error { for _, attacher := range fakeVolumePlugin.GetAttachers() { actualCallCount := attacher.GetMountDeviceCallCount() if actualCallCount != 0 { return fmt.Errorf( "At least one attacher has non-zero MountDeviceCallCount: <%v>.", actualCallCount) } } return nil } // VerifySetUpCallCount ensures that at least one of the Mounters for this // plugin has the expectedSetUpCallCount number of calls. Otherwise it returns // an error. func VerifySetUpCallCount( expectedSetUpCallCount int, fakeVolumePlugin *FakeVolumePlugin) error { for _, mounter := range fakeVolumePlugin.GetMounters() { actualCallCount := mounter.GetSetUpCallCount() if actualCallCount >= expectedSetUpCallCount { return nil } } return fmt.Errorf( "No Mounters have expected SetUpCallCount. Expected: <%v>.", expectedSetUpCallCount) } // VerifyZeroSetUpCallCount ensures that all Mounters for this plugin have a // zero SetUpCallCount. Otherwise it returns an error. func VerifyZeroSetUpCallCount(fakeVolumePlugin *FakeVolumePlugin) error { for _, mounter := range fakeVolumePlugin.GetMounters() { actualCallCount := mounter.GetSetUpCallCount() if actualCallCount != 0 { return fmt.Errorf( "At least one mounter has non-zero SetUpCallCount: <%v>.", actualCallCount) } } return nil } // VerifyTearDownCallCount ensures that at least one of the Unounters for this // plugin has the expectedTearDownCallCount number of calls. Otherwise it // returns an error. func VerifyTearDownCallCount( expectedTearDownCallCount int, fakeVolumePlugin *FakeVolumePlugin) error { for _, unmounter := range fakeVolumePlugin.GetUnmounters() { actualCallCount := unmounter.GetTearDownCallCount() if actualCallCount >= expectedTearDownCallCount { return nil } } return fmt.Errorf( "No Unmounters have expected SetUpCallCount. Expected: <%v>.", expectedTearDownCallCount) } // VerifyZeroTearDownCallCount ensures that all Mounters for this plugin have a // zero TearDownCallCount. Otherwise it returns an error. func VerifyZeroTearDownCallCount(fakeVolumePlugin *FakeVolumePlugin) error { for _, mounter := range fakeVolumePlugin.GetMounters() { actualCallCount := mounter.GetTearDownCallCount() if actualCallCount != 0 { return fmt.Errorf( "At least one mounter has non-zero TearDownCallCount: <%v>.", actualCallCount) } } return nil } // VerifyDetachCallCount ensures that at least one of the Attachers for this // plugin has the expectedDetachCallCount number of calls. Otherwise it returns // an error. func VerifyDetachCallCount( expectedDetachCallCount int, fakeVolumePlugin *FakeVolumePlugin) error { for _, detacher := range fakeVolumePlugin.GetDetachers() { actualCallCount := detacher.GetDetachCallCount() if actualCallCount == expectedDetachCallCount { return nil } } return fmt.Errorf( "No Detachers have expected DetachCallCount. Expected: <%v>.", expectedDetachCallCount) } // VerifyZeroDetachCallCount ensures that all Detachers for this plugin have a // zero DetachCallCount. Otherwise it returns an error. func VerifyZeroDetachCallCount(fakeVolumePlugin *FakeVolumePlugin) error { for _, detacher := range fakeVolumePlugin.GetDetachers() { actualCallCount := detacher.GetDetachCallCount() if actualCallCount != 0 { return fmt.Errorf( "At least one detacher has non-zero DetachCallCount: <%v>.", actualCallCount) } } return nil } // GetTestVolumePluginMgr creates, initializes, and returns a test volume plugin // manager and fake volume plugin using a fake volume host. func GetTestVolumePluginMgr( t *testing.T) (*VolumePluginMgr, *FakeVolumePlugin) { v := NewFakeVolumeHost( "", /* rootDir */ nil, /* kubeClient */ nil, /* plugins */ ) plugins := ProbeVolumePlugins(VolumeConfig{}) if err := v.pluginMgr.InitPlugins(plugins, v); err != nil { t.Fatal(err) } return &v.pluginMgr, plugins[0].(*FakeVolumePlugin) } // CreateTestPVC returns a provisionable PVC for tests func CreateTestPVC(capacity string, accessModes []v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim { claim := v1.PersistentVolumeClaim{ ObjectMeta: v1.ObjectMeta{ Name: "dummy", Namespace: "default", }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: accessModes, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceName(v1.ResourceStorage): resource.MustParse(capacity), }, }, }, } return &claim }