// +build linux // // projectquota.go - implements XFS project quota controls // for setting quota limits on a newly created directory. // It currently supports the legacy XFS specific ioctls. // // TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR // for both xfs/ext4 for kernel version >= v4.5 // package quota /* #include #include #include #include #include #ifndef FS_XFLAG_PROJINHERIT struct fsxattr { __u32 fsx_xflags; __u32 fsx_extsize; __u32 fsx_nextents; __u32 fsx_projid; unsigned char fsx_pad[12]; }; #define FS_XFLAG_PROJINHERIT 0x00000200 #endif #ifndef FS_IOC_FSGETXATTR #define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) #endif #ifndef FS_IOC_FSSETXATTR #define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) #endif #ifndef PRJQUOTA #define PRJQUOTA 2 #endif #ifndef XFS_PROJ_QUOTA #define XFS_PROJ_QUOTA 2 #endif #ifndef Q_XSETPQLIM #define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA) #endif #ifndef Q_XGETPQUOTA #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) #endif */ import "C" import ( "fmt" "io/ioutil" "path" "path/filepath" "unsafe" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) // Quota limit params - currently we only control blocks hard limit type Quota struct { Size uint64 } // Control - Context to be used by storage driver (e.g. overlay) // who wants to apply project quotas to container dirs type Control struct { backingFsBlockDev string nextProjectID uint32 quotas map[string]uint32 } // NewControl - initialize project quota support. // Test to make sure that quota can be set on a test dir and find // the first project id to be used for the next container create. // // Returns nil (and error) if project quota is not supported. // // First get the project id of the home directory. // This test will fail if the backing fs is not xfs. // // xfs_quota tool can be used to assign a project id to the driver home directory, e.g.: // echo 999:/var/lib/containers/storage/overlay >> /etc/projects // echo storage:999 >> /etc/projid // xfs_quota -x -c 'project -s storage' / // // In that case, the home directory project id will be used as a "start offset" // and all containers will be assigned larger project ids (e.g. >= 1000). // This is a way to prevent xfs_quota management from conflicting with containers/storage. // // Then try to create a test directory with the next project id and set a quota // on it. If that works, continue to scan existing containers to map allocated // project ids. // func NewControl(basePath string) (*Control, error) { // // Get project id of parent dir as minimal id to be used by driver // minProjectID, err := getProjectID(basePath) if err != nil { return nil, err } minProjectID++ // // create backing filesystem device node // backingFsBlockDev, err := makeBackingFsDev(basePath) if err != nil { return nil, err } // // Test if filesystem supports project quotas by trying to set // a quota on the first available project id // quota := Quota{ Size: 0, } if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil { return nil, err } q := Control{ backingFsBlockDev: backingFsBlockDev, nextProjectID: minProjectID + 1, quotas: make(map[string]uint32), } // // get first project id to be used for next container // err = q.findNextProjectID(basePath) if err != nil { return nil, err } logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID) return &q, nil } // SetQuota - assign a unique project id to directory and set the quota limits // for that project id func (q *Control) SetQuota(targetPath string, quota Quota) error { projectID, ok := q.quotas[targetPath] if !ok { projectID = q.nextProjectID // // assign project id to new container directory // err := setProjectID(targetPath, projectID) if err != nil { return err } q.quotas[targetPath] = projectID q.nextProjectID++ } // // set the quota limit for the container's project id // logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID) return setProjectQuota(q.backingFsBlockDev, projectID, quota) } // setProjectQuota - set the quota for project id on xfs block device func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error { var d C.fs_disk_quota_t d.d_version = C.FS_DQUOT_VERSION d.d_id = C.__u32(projectID) d.d_flags = C.XFS_PROJ_QUOTA d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT d.d_blk_hardlimit = C.__u64(quota.Size / 512) d.d_blk_softlimit = d.d_blk_hardlimit var cs = C.CString(backingFsBlockDev) defer C.free(unsafe.Pointer(cs)) _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), uintptr(unsafe.Pointer(&d)), 0, 0) if errno != 0 { return fmt.Errorf("Failed to set quota limit for projid %d on %s: %v", projectID, backingFsBlockDev, errno.Error()) } return nil } // GetQuota - get the quota limits of a directory that was configured with SetQuota func (q *Control) GetQuota(targetPath string, quota *Quota) error { projectID, ok := q.quotas[targetPath] if !ok { return fmt.Errorf("quota not found for path : %s", targetPath) } // // get the quota limit for the container's project id // var d C.fs_disk_quota_t var cs = C.CString(q.backingFsBlockDev) defer C.free(unsafe.Pointer(cs)) _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA, uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)), uintptr(unsafe.Pointer(&d)), 0, 0) if errno != 0 { return fmt.Errorf("Failed to get quota limit for projid %d on %s: %v", projectID, q.backingFsBlockDev, errno.Error()) } quota.Size = uint64(d.d_blk_hardlimit) * 512 return nil } // getProjectID - get the project id of path on xfs func getProjectID(targetPath string) (uint32, error) { dir, err := openDir(targetPath) if err != nil { return 0, err } defer closeDir(dir) var fsx C.struct_fsxattr _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, uintptr(unsafe.Pointer(&fsx))) if errno != 0 { return 0, fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) } return uint32(fsx.fsx_projid), nil } // setProjectID - set the project id of path on xfs func setProjectID(targetPath string, projectID uint32) error { dir, err := openDir(targetPath) if err != nil { return err } defer closeDir(dir) var fsx C.struct_fsxattr _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, uintptr(unsafe.Pointer(&fsx))) if errno != 0 { return fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) } fsx.fsx_projid = C.__u32(projectID) fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, uintptr(unsafe.Pointer(&fsx))) if errno != 0 { return fmt.Errorf("Failed to set projid for %s: %v", targetPath, errno.Error()) } return nil } // findNextProjectID - find the next project id to be used for containers // by scanning driver home directory to find used project ids func (q *Control) findNextProjectID(home string) error { files, err := ioutil.ReadDir(home) if err != nil { return fmt.Errorf("read directory failed : %s", home) } for _, file := range files { if !file.IsDir() { continue } path := filepath.Join(home, file.Name()) projid, err := getProjectID(path) if err != nil { return err } if projid > 0 { q.quotas[path] = projid } if q.nextProjectID <= projid { q.nextProjectID = projid + 1 } } return nil } func free(p *C.char) { C.free(unsafe.Pointer(p)) } func openDir(path string) (*C.DIR, error) { Cpath := C.CString(path) defer free(Cpath) dir := C.opendir(Cpath) if dir == nil { return nil, fmt.Errorf("Can't open dir") } return dir, nil } func closeDir(dir *C.DIR) { if dir != nil { C.closedir(dir) } } func getDirFd(dir *C.DIR) uintptr { return uintptr(C.dirfd(dir)) } // Get the backing block device of the driver home directory // and create a block device node under the home directory // to be used by quotactl commands func makeBackingFsDev(home string) (string, error) { var stat unix.Stat_t if err := unix.Stat(home, &stat); err != nil { return "", err } backingFsBlockDev := path.Join(home, "backingFsBlockDev") // Re-create just in case someone copied the home directory over to a new device unix.Unlink(backingFsBlockDev) if err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev)); err != nil { return "", fmt.Errorf("Failed to mknod %s: %v", backingFsBlockDev, err) } return backingFsBlockDev, nil }