From 64ffdbc701e470881023256b408c2d369717ac00 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Tue, 21 Apr 2015 18:14:59 -0400 Subject: [PATCH 1/3] devicemapper: Add helper functions to allow deferred device removal A lot of time device mapper devices leak across mount namespace which docker does not know about and when docker tries to deactivate/delete device, operation fails as device is open in some mount namespace. Create a mechanism where one can defer the device deactivation/deletion so that docker operation does not fail and device automatically goes away when last reference to it is dropped. Signed-off-by: Vivek Goyal --- devicemapper/devmapper.go | 20 +++++++++++++++++++ devicemapper/devmapper_wrapper.go | 1 + .../devmapper_wrapper_deferred_remove.go | 15 ++++++++++++++ .../devmapper_wrapper_no_deferred_remove.go | 10 ++++++++++ 4 files changed, 46 insertions(+) create mode 100644 devicemapper/devmapper_wrapper_deferred_remove.go create mode 100644 devicemapper/devmapper_wrapper_no_deferred_remove.go diff --git a/devicemapper/devmapper.go b/devicemapper/devmapper.go index bb89f7f..42876d6 100644 --- a/devicemapper/devmapper.go +++ b/devicemapper/devmapper.go @@ -55,6 +55,7 @@ var ( ErrTaskGetDeps = errors.New("dm_task_get_deps failed") ErrTaskGetInfo = errors.New("dm_task_get_info failed") ErrTaskGetDriverVersion = errors.New("dm_task_get_driver_version failed") + ErrTaskDeferredRemove = errors.New("dm_task_deferred_remove failed") ErrTaskSetCookie = errors.New("dm_task_set_cookie failed") ErrNilCookie = errors.New("cookie ptr can't be nil") ErrAttachLoopbackDevice = errors.New("loopback mounting failed") @@ -371,6 +372,25 @@ func RemoveDevice(name string) error { return nil } +func RemoveDeviceDeferred(name string) error { + logrus.Debugf("[devmapper] RemoveDeviceDeferred START(%s)", name) + defer logrus.Debugf("[devmapper] RemoveDeviceDeferred END(%s)", name) + task, err := TaskCreateNamed(DeviceRemove, name) + if task == nil { + return err + } + + if err := DmTaskDeferredRemove(task.unmanaged); err != 1 { + return ErrTaskDeferredRemove + } + + if err = task.Run(); err != nil { + return fmt.Errorf("Error running RemoveDeviceDeferred %s", err) + } + + return nil +} + func GetBlockDeviceSize(file *os.File) (uint64, error) { size, err := ioctlBlkGetSize64(file.Fd()) if err != nil { diff --git a/devicemapper/devmapper_wrapper.go b/devicemapper/devmapper_wrapper.go index e436cca..fc841d9 100644 --- a/devicemapper/devmapper_wrapper.go +++ b/devicemapper/devmapper_wrapper.go @@ -112,6 +112,7 @@ var ( DmUdevGetSyncSupport = dmUdevGetSyncSupportFct DmCookieSupported = dmCookieSupportedFct LogWithErrnoInit = logWithErrnoInitFct + DmTaskDeferredRemove = dmTaskDeferredRemoveFct ) func free(p *C.char) { diff --git a/devicemapper/devmapper_wrapper_deferred_remove.go b/devicemapper/devmapper_wrapper_deferred_remove.go new file mode 100644 index 0000000..3d52f3f --- /dev/null +++ b/devicemapper/devmapper_wrapper_deferred_remove.go @@ -0,0 +1,15 @@ +// +build linux,!libdm_no_deferred_remove + +package devicemapper + +/* +#cgo LDFLAGS: -L. -ldevmapper +#include +*/ +import "C" + +const LibraryDeferredRemovalSupport = true + +func dmTaskDeferredRemoveFct(task *CDmTask) int { + return int(C.dm_task_deferred_remove((*C.struct_dm_task)(task))) +} diff --git a/devicemapper/devmapper_wrapper_no_deferred_remove.go b/devicemapper/devmapper_wrapper_no_deferred_remove.go new file mode 100644 index 0000000..6366065 --- /dev/null +++ b/devicemapper/devmapper_wrapper_no_deferred_remove.go @@ -0,0 +1,10 @@ +// +build linux,libdm_no_deferred_remove + +package devicemapper + +const LibraryDeferredRemovalSupport = false + +func dmTaskDeferredRemoveFct(task *CDmTask) int { + // Error. Nobody should be calling it. + return -1 +} From 48536e29e50b70f7f513b741a77146df00288c91 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Tue, 21 Apr 2015 18:14:59 -0400 Subject: [PATCH 2/3] devicemapper: Create helpers to cancel deferred deactivation If a device has been scheduled for deferred deactivation and container is started again and we need to activate device again, we need to cancel the deferred deactivation which is already scheduled on the device. Create a method for the same. Signed-off-by: Vivek Goyal --- devicemapper/devmapper.go | 32 ++++++++++++++++++++++++++++++++ devicemapper/devmapper_log.go | 4 ++++ 2 files changed, 36 insertions(+) diff --git a/devicemapper/devmapper.go b/devicemapper/devmapper.go index 42876d6..04ad912 100644 --- a/devicemapper/devmapper.go +++ b/devicemapper/devmapper.go @@ -70,9 +70,11 @@ var ( ErrLoopbackSetCapacity = errors.New("Unable set loopback capacity") ErrBusy = errors.New("Device is Busy") ErrDeviceIdExists = errors.New("Device Id Exists") + ErrEnxio = errors.New("No such device or address") dmSawBusy bool dmSawExist bool + dmSawEnxio bool // No Such Device or Address ) type ( @@ -391,6 +393,36 @@ func RemoveDeviceDeferred(name string) error { return nil } +// Useful helper for cleanup +func CancelDeferredRemove(deviceName string) error { + task, err := TaskCreateNamed(DeviceTargetMsg, deviceName) + if task == nil { + return err + } + + if err := task.SetSector(0); err != nil { + return fmt.Errorf("Can't set sector %s", err) + } + + if err := task.SetMessage(fmt.Sprintf("@cancel_deferred_remove")); err != nil { + return fmt.Errorf("Can't set message %s", err) + } + + dmSawBusy = false + dmSawEnxio = false + if err := task.Run(); err != nil { + // A device might be being deleted already + if dmSawBusy { + return ErrBusy + } else if dmSawEnxio { + return ErrEnxio + } + return fmt.Errorf("Error running CancelDeferredRemove %s", err) + + } + return nil +} + func GetBlockDeviceSize(file *os.File) (uint64, error) { size, err := ioctlBlkGetSize64(file.Fd()) if err != nil { diff --git a/devicemapper/devmapper_log.go b/devicemapper/devmapper_log.go index d6550bd..f66a208 100644 --- a/devicemapper/devmapper_log.go +++ b/devicemapper/devmapper_log.go @@ -22,6 +22,10 @@ func DevmapperLogCallback(level C.int, file *C.char, line C.int, dm_errno_or_cla if strings.Contains(msg, "File exists") { dmSawExist = true } + + if strings.Contains(msg, "No such device or address") { + dmSawEnxio = true + } } if dmLogger != nil { From e2bca8900c8c77491fba8fb829ea0fc6ccb03e8f Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Tue, 21 Apr 2015 18:14:59 -0400 Subject: [PATCH 3/3] devicemapper: Create a method to get device info with deferred remove field Deferred reove functionality was added to library later. So in old version of library it did not report deferred_remove field. Create a new function which also gets deferred_remove field and it will be called only on newer version of library. Signed-off-by: Vivek Goyal --- devicemapper/devmapper.go | 40 ++++++++++++---- devicemapper/devmapper_wrapper.go | 47 ++++++++++--------- .../devmapper_wrapper_deferred_remove.go | 18 +++++++ .../devmapper_wrapper_no_deferred_remove.go | 4 ++ 4 files changed, 76 insertions(+), 33 deletions(-) diff --git a/devicemapper/devmapper.go b/devicemapper/devmapper.go index 04ad912..e7f17b8 100644 --- a/devicemapper/devmapper.go +++ b/devicemapper/devmapper.go @@ -87,16 +87,17 @@ type ( Device []uint64 } Info struct { - Exists int - Suspended int - LiveTable int - InactiveTable int - OpenCount int32 - EventNr uint32 - Major uint32 - Minor uint32 - ReadOnly int - TargetCount int32 + Exists int + Suspended int + LiveTable int + InactiveTable int + OpenCount int32 + EventNr uint32 + Major uint32 + Minor uint32 + ReadOnly int + TargetCount int32 + DeferredRemove int } TaskType int AddNodeType int @@ -222,6 +223,14 @@ func (t *Task) GetInfo() (*Info, error) { return info, nil } +func (t *Task) GetInfoWithDeferred() (*Info, error) { + info := &Info{} + if res := DmTaskGetInfoWithDeferred(t.unmanaged, info); res != 1 { + return nil, ErrTaskGetInfo + } + return info, nil +} + func (t *Task) GetDriverVersion() (string, error) { res := DmTaskGetDriverVersion(t.unmanaged) if res == "" { @@ -531,6 +540,17 @@ func GetInfo(name string) (*Info, error) { return task.GetInfo() } +func GetInfoWithDeferred(name string) (*Info, error) { + task, err := TaskCreateNamed(DeviceInfo, name) + if task == nil { + return nil, err + } + if err := task.Run(); err != nil { + return nil, err + } + return task.GetInfoWithDeferred() +} + func GetDriverVersion() (string, error) { task := TaskCreate(DeviceVersion) if task == nil { diff --git a/devicemapper/devmapper_wrapper.go b/devicemapper/devmapper_wrapper.go index fc841d9..87c2003 100644 --- a/devicemapper/devmapper_wrapper.go +++ b/devicemapper/devmapper_wrapper.go @@ -90,29 +90,30 @@ const ( ) var ( - DmGetLibraryVersion = dmGetLibraryVersionFct - DmGetNextTarget = dmGetNextTargetFct - DmLogInitVerbose = dmLogInitVerboseFct - DmSetDevDir = dmSetDevDirFct - DmTaskAddTarget = dmTaskAddTargetFct - DmTaskCreate = dmTaskCreateFct - DmTaskDestroy = dmTaskDestroyFct - DmTaskGetDeps = dmTaskGetDepsFct - DmTaskGetInfo = dmTaskGetInfoFct - DmTaskGetDriverVersion = dmTaskGetDriverVersionFct - DmTaskRun = dmTaskRunFct - DmTaskSetAddNode = dmTaskSetAddNodeFct - DmTaskSetCookie = dmTaskSetCookieFct - DmTaskSetMessage = dmTaskSetMessageFct - DmTaskSetName = dmTaskSetNameFct - DmTaskSetRo = dmTaskSetRoFct - DmTaskSetSector = dmTaskSetSectorFct - DmUdevWait = dmUdevWaitFct - DmUdevSetSyncSupport = dmUdevSetSyncSupportFct - DmUdevGetSyncSupport = dmUdevGetSyncSupportFct - DmCookieSupported = dmCookieSupportedFct - LogWithErrnoInit = logWithErrnoInitFct - DmTaskDeferredRemove = dmTaskDeferredRemoveFct + DmGetLibraryVersion = dmGetLibraryVersionFct + DmGetNextTarget = dmGetNextTargetFct + DmLogInitVerbose = dmLogInitVerboseFct + DmSetDevDir = dmSetDevDirFct + DmTaskAddTarget = dmTaskAddTargetFct + DmTaskCreate = dmTaskCreateFct + DmTaskDestroy = dmTaskDestroyFct + DmTaskGetDeps = dmTaskGetDepsFct + DmTaskGetInfo = dmTaskGetInfoFct + DmTaskGetDriverVersion = dmTaskGetDriverVersionFct + DmTaskRun = dmTaskRunFct + DmTaskSetAddNode = dmTaskSetAddNodeFct + DmTaskSetCookie = dmTaskSetCookieFct + DmTaskSetMessage = dmTaskSetMessageFct + DmTaskSetName = dmTaskSetNameFct + DmTaskSetRo = dmTaskSetRoFct + DmTaskSetSector = dmTaskSetSectorFct + DmUdevWait = dmUdevWaitFct + DmUdevSetSyncSupport = dmUdevSetSyncSupportFct + DmUdevGetSyncSupport = dmUdevGetSyncSupportFct + DmCookieSupported = dmCookieSupportedFct + LogWithErrnoInit = logWithErrnoInitFct + DmTaskDeferredRemove = dmTaskDeferredRemoveFct + DmTaskGetInfoWithDeferred = dmTaskGetInfoWithDeferredFct ) func free(p *C.char) { diff --git a/devicemapper/devmapper_wrapper_deferred_remove.go b/devicemapper/devmapper_wrapper_deferred_remove.go index 3d52f3f..ced482c 100644 --- a/devicemapper/devmapper_wrapper_deferred_remove.go +++ b/devicemapper/devmapper_wrapper_deferred_remove.go @@ -13,3 +13,21 @@ const LibraryDeferredRemovalSupport = true func dmTaskDeferredRemoveFct(task *CDmTask) int { return int(C.dm_task_deferred_remove((*C.struct_dm_task)(task))) } + +func dmTaskGetInfoWithDeferredFct(task *CDmTask, info *Info) int { + Cinfo := C.struct_dm_info{} + defer func() { + info.Exists = int(Cinfo.exists) + info.Suspended = int(Cinfo.suspended) + info.LiveTable = int(Cinfo.live_table) + info.InactiveTable = int(Cinfo.inactive_table) + info.OpenCount = int32(Cinfo.open_count) + info.EventNr = uint32(Cinfo.event_nr) + info.Major = uint32(Cinfo.major) + info.Minor = uint32(Cinfo.minor) + info.ReadOnly = int(Cinfo.read_only) + info.TargetCount = int32(Cinfo.target_count) + info.DeferredRemove = int(Cinfo.deferred_remove) + }() + return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo)) +} diff --git a/devicemapper/devmapper_wrapper_no_deferred_remove.go b/devicemapper/devmapper_wrapper_no_deferred_remove.go index 6366065..16631bf 100644 --- a/devicemapper/devmapper_wrapper_no_deferred_remove.go +++ b/devicemapper/devmapper_wrapper_no_deferred_remove.go @@ -8,3 +8,7 @@ func dmTaskDeferredRemoveFct(task *CDmTask) int { // Error. Nobody should be calling it. return -1 } + +func dmTaskGetInfoWithDeferredFct(task *CDmTask, info *Info) int { + return -1 +}