mirror of
https://github.com/vbatts/imgsrv.git
synced 2024-11-23 16:45:39 +00:00
dbutil: abstract out mongo Handler
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
This commit is contained in:
parent
1b64b8a3c5
commit
efa8ac6511
GPG key ID:
10937E57733F1362
3 changed files with 273 additions and 266 deletions
273
dbutil/dbutil.go
273
dbutil/dbutil.go
|
|
@ -1,258 +1,33 @@
|
|||
package dbutil
|
||||
|
||||
import (
|
||||
"github.com/vbatts/imgsrv/hash"
|
||||
"github.com/vbatts/imgsrv/types"
|
||||
"labix.org/v2/mgo"
|
||||
"labix.org/v2/mgo/bson"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
DEFAULT_DB_NAME = "filesrv"
|
||||
)
|
||||
// Handles are all the register backing Handlers
|
||||
var Handles map[string]Handler
|
||||
|
||||
type Util struct {
|
||||
Seed string // mongo host seed to Dial into
|
||||
User string // mongo credentials, if needed
|
||||
Pass string // mongo credentials, if needed
|
||||
DbName string // mongo database name, if needed
|
||||
Session *mgo.Session
|
||||
FileDb *mgo.Database
|
||||
Gfs *mgo.GridFS
|
||||
}
|
||||
|
||||
func (u *Util) Init() error {
|
||||
var err error
|
||||
u.Session, err = mgo.Dial(u.Seed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(u.DbName) > 0 {
|
||||
u.FileDb = u.Session.DB(u.DbName)
|
||||
} else {
|
||||
u.FileDb = u.Session.DB(DEFAULT_DB_NAME)
|
||||
}
|
||||
|
||||
if len(u.User) > 0 && len(u.Pass) > 0 {
|
||||
err = u.FileDb.Login(u.User, u.Pass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
u.Gfs = u.FileDb.GridFS("fs")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u Util) Close() {
|
||||
u.Session.Close()
|
||||
}
|
||||
|
||||
/*
|
||||
pass through for GridFs
|
||||
*/
|
||||
func (u Util) Open(filename string) (file *mgo.GridFile, err error) {
|
||||
return u.Gfs.Open(strings.ToLower(filename))
|
||||
}
|
||||
|
||||
/*
|
||||
pass through for GridFs
|
||||
*/
|
||||
func (u Util) Create(filename string) (file *mgo.GridFile, err error) {
|
||||
return u.Gfs.Create(strings.ToLower(filename))
|
||||
}
|
||||
|
||||
/*
|
||||
pass through for GridFs
|
||||
*/
|
||||
func (u Util) Remove(filename string) (err error) {
|
||||
return u.Gfs.Remove(strings.ToLower(filename))
|
||||
}
|
||||
|
||||
/*
|
||||
Find files by their MD5 checksum
|
||||
*/
|
||||
func (u Util) FindFilesByMd5(md5 string) (files []types.File, err error) {
|
||||
err = u.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
/*
|
||||
match for file name
|
||||
*/
|
||||
func (u Util) FindFilesByName(filename string) (files []types.File, err error) {
|
||||
err = u.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
/*
|
||||
Case-insensitive pattern match for file name
|
||||
*/
|
||||
func (u Util) FindFilesByPatt(filename_pat string) (files []types.File, err error) {
|
||||
err = u.Gfs.Find(bson.M{"filename": bson.M{"$regex": filename_pat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
/*
|
||||
Case-insensitive pattern match for file name
|
||||
*/
|
||||
func (u Util) FindFilesByKeyword(keyword string) (files []types.File, err error) {
|
||||
err = u.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
/*
|
||||
Get all the files.
|
||||
|
||||
pass -1 for all files
|
||||
*/
|
||||
func (u Util) GetFiles(limit int) (files []types.File, err error) {
|
||||
//files = []types.File{}
|
||||
if limit == -1 {
|
||||
err = u.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
|
||||
} else {
|
||||
err = u.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
|
||||
}
|
||||
return files, err
|
||||
}
|
||||
|
||||
/*
|
||||
Count the filename matches
|
||||
*/
|
||||
func (u Util) CountFiles(filename string) (count int, err error) {
|
||||
query := u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
|
||||
return query.Count()
|
||||
}
|
||||
|
||||
/*
|
||||
Get one file back, by searching by file name
|
||||
*/
|
||||
func (u Util) GetFileByFilename(filename string) (this_file types.File, err error) {
|
||||
err = u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&this_file)
|
||||
if err != nil {
|
||||
return this_file, err
|
||||
}
|
||||
return this_file, nil
|
||||
}
|
||||
|
||||
func (u Util) GetFileRandom() (this_file types.File, err error) {
|
||||
r := hash.Rand64()
|
||||
err = u.Gfs.Find(bson.M{"random": bson.M{"$gt": r}}).One(&this_file)
|
||||
if err != nil {
|
||||
return this_file, err
|
||||
}
|
||||
if len(this_file.Md5) == 0 {
|
||||
err = u.Gfs.Find(bson.M{"random": bson.M{"$lt": r}}).One(&this_file)
|
||||
}
|
||||
if err != nil {
|
||||
return this_file, err
|
||||
}
|
||||
return this_file, nil
|
||||
}
|
||||
|
||||
/*
|
||||
Check whether this types.File filename is on Mongo
|
||||
*/
|
||||
func (u Util) HasFileByFilename(filename string) (exists bool, err error) {
|
||||
c, err := u.CountFiles(filename)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
exists = (c > 0)
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (u Util) HasFileByMd5(md5 string) (exists bool, err error) {
|
||||
c, err := u.Gfs.Find(bson.M{"md5": md5}).Count()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
exists = (c > 0)
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (u Util) HasFileByKeyword(keyword string) (exists bool, err error) {
|
||||
c, err := u.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
exists = (c > 0)
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
/*
|
||||
get a list of file extensions and their frequency count
|
||||
*/
|
||||
func (u Util) GetExtensions() (kp []types.IdCount, err error) {
|
||||
job := &mgo.MapReduce{
|
||||
Map: `
|
||||
function() {
|
||||
if (!this.filename) {
|
||||
return;
|
||||
}
|
||||
|
||||
s = this.filename.split(".")
|
||||
ext = s[s.length - 1] // get the last segment of the split
|
||||
emit(ext,1);
|
||||
}
|
||||
`,
|
||||
Reduce: `
|
||||
function(previous, current) {
|
||||
var count = 0;
|
||||
|
||||
for (index in current) {
|
||||
count += current[index];
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
`,
|
||||
}
|
||||
if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
|
||||
return kp, err
|
||||
}
|
||||
// Less than effecient, but cleanest place to put this
|
||||
for i := range kp {
|
||||
kp[i].Root = "ext" // for extension. Maps to /ext/
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
/*
|
||||
get a list of keywords and their frequency count
|
||||
*/
|
||||
func (u Util) GetKeywords() (kp []types.IdCount, err error) {
|
||||
job := &mgo.MapReduce{
|
||||
Map: `
|
||||
function() {
|
||||
if (!this.metadata.keywords) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (index in this.metadata.keywords) {
|
||||
emit(this.metadata.keywords[index], 1);
|
||||
}
|
||||
}
|
||||
`,
|
||||
Reduce: `
|
||||
function(previous, current) {
|
||||
var count = 0;
|
||||
|
||||
for (index in current) {
|
||||
count += current[index];
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
`,
|
||||
}
|
||||
if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
|
||||
return kp, err
|
||||
}
|
||||
// Less than effecient, but cleanest place to put this
|
||||
for i := range kp {
|
||||
kp[i].Root = "k" // for keyword. Maps to /k/
|
||||
}
|
||||
return kp, nil
|
||||
// Handler is the means of getting "files" from the backing database
|
||||
type Handler interface {
|
||||
Init(config interface{}) error
|
||||
Close() error
|
||||
|
||||
Open(filename string) (*mgo.GridFile, error)
|
||||
Create(filename string) (*mgo.GridFile, error)
|
||||
Remove(filename string) error
|
||||
|
||||
//HasFileByMd5(md5 string) (exists bool, err error)
|
||||
//HasFileByKeyword(keyword string) (exists bool, err error)
|
||||
HasFileByFilename(filename string) (exists bool, err error)
|
||||
FindFilesByKeyword(keyword string) (files []types.File, err error)
|
||||
FindFilesByMd5(md5 string) (files []types.File, err error)
|
||||
FindFilesByPatt(filenamePat string) (files []types.File, err error)
|
||||
|
||||
CountFiles(filename string) (int, error)
|
||||
|
||||
GetFiles(limit int) (files []types.File, err error)
|
||||
GetFileByFilename(filename string) (types.File, error)
|
||||
GetExtensions() (kp []types.IdCount, err error)
|
||||
GetKeywords() (kp []types.IdCount, err error)
|
||||
}
|
||||
|
|
|
|||
228
dbutil/mongo/handle.go
Normal file
228
dbutil/mongo/handle.go
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
package mongo
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/vbatts/imgsrv/dbutil"
|
||||
"github.com/vbatts/imgsrv/types"
|
||||
"labix.org/v2/mgo"
|
||||
"labix.org/v2/mgo/bson"
|
||||
)
|
||||
|
||||
func init() {
|
||||
dbutil.Handles["mongo"] = &mongoHandle{}
|
||||
}
|
||||
|
||||
const defaultDbName = "filesrv"
|
||||
|
||||
type dbConfig struct {
|
||||
Seed string // mongo host seed to Dial into
|
||||
User string // mongo credentials, if needed
|
||||
Pass string // mongo credentials, if needed
|
||||
DbName string // mongo database name, if needed
|
||||
}
|
||||
|
||||
type mongoHandle struct {
|
||||
config dbConfig
|
||||
Session *mgo.Session
|
||||
FileDb *mgo.Database
|
||||
Gfs *mgo.GridFS
|
||||
}
|
||||
|
||||
func (h *mongoHandle) Init(config interface{}) (err error) {
|
||||
h.config = config.(dbConfig)
|
||||
|
||||
h.Session, err = mgo.Dial(h.config.Seed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(h.config.DbName) > 0 {
|
||||
h.FileDb = h.Session.DB(h.config.DbName)
|
||||
} else {
|
||||
h.FileDb = h.Session.DB(defaultDbName)
|
||||
}
|
||||
|
||||
if len(h.config.User) > 0 && len(h.config.Pass) > 0 {
|
||||
err = h.FileDb.Login(h.config.User, h.config.Pass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
h.Gfs = h.FileDb.GridFS("fs")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h mongoHandle) Close() error {
|
||||
h.Session.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// pass through for GridFs
|
||||
func (h mongoHandle) Open(filename string) (file *mgo.GridFile, err error) {
|
||||
return h.Gfs.Open(strings.ToLower(filename))
|
||||
}
|
||||
|
||||
// pass through for GridFs
|
||||
func (h mongoHandle) Create(filename string) (file *mgo.GridFile, err error) {
|
||||
return h.Gfs.Create(strings.ToLower(filename))
|
||||
}
|
||||
|
||||
// pass through for GridFs
|
||||
func (h mongoHandle) Remove(filename string) (err error) {
|
||||
return h.Gfs.Remove(strings.ToLower(filename))
|
||||
}
|
||||
|
||||
// Find files by their MD5 checksum
|
||||
func (h mongoHandle) FindFilesByMd5(md5 string) (files []types.File, err error) {
|
||||
err = h.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
// match for file name
|
||||
// XXX this is not used
|
||||
func (h mongoHandle) FindFilesByName(filename string) (files []types.File, err error) {
|
||||
err = h.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
// Case-insensitive pattern match for file name
|
||||
func (h mongoHandle) FindFilesByPatt(filenamePat string) (files []types.File, err error) {
|
||||
err = h.Gfs.Find(bson.M{"filename": bson.M{"$regex": filenamePat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
// Case-insensitive pattern match for file name
|
||||
func (h mongoHandle) FindFilesByKeyword(keyword string) (files []types.File, err error) {
|
||||
err = h.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
|
||||
return files, err
|
||||
}
|
||||
|
||||
// Get all the files.
|
||||
// Pass -1 for all files.
|
||||
func (h mongoHandle) GetFiles(limit int) (files []types.File, err error) {
|
||||
//files = []types.File{}
|
||||
if limit == -1 {
|
||||
err = h.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
|
||||
} else {
|
||||
err = h.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
|
||||
}
|
||||
return files, err
|
||||
}
|
||||
|
||||
// Count the filename matches
|
||||
func (h mongoHandle) CountFiles(filename string) (count int, err error) {
|
||||
query := h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
|
||||
return query.Count()
|
||||
}
|
||||
|
||||
// Get one file back, by searching by file name
|
||||
func (h mongoHandle) GetFileByFilename(filename string) (thisFile types.File, err error) {
|
||||
err = h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&thisFile)
|
||||
if err != nil {
|
||||
return thisFile, err
|
||||
}
|
||||
return thisFile, nil
|
||||
}
|
||||
|
||||
// Check whether this types.File filename is on Mongo
|
||||
func (h mongoHandle) HasFileByFilename(filename string) (exists bool, err error) {
|
||||
c, err := h.CountFiles(filename)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
exists = (c > 0)
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// XXX this is not used
|
||||
func (h mongoHandle) HasFileByMd5(md5 string) (exists bool, err error) {
|
||||
c, err := h.Gfs.Find(bson.M{"md5": md5}).Count()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
exists = (c > 0)
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// XXX this is not used
|
||||
func (h mongoHandle) HasFileByKeyword(keyword string) (exists bool, err error) {
|
||||
c, err := h.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
exists = (c > 0)
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// get a list of file extensions and their frequency count
|
||||
func (h mongoHandle) GetExtensions() (kp []types.IdCount, err error) {
|
||||
job := &mgo.MapReduce{
|
||||
Map: `
|
||||
function() {
|
||||
if (!this.filename) {
|
||||
return;
|
||||
}
|
||||
|
||||
s = this.filename.split(".")
|
||||
ext = s[s.length - 1] // get the last segment of the split
|
||||
emit(ext,1);
|
||||
}
|
||||
`,
|
||||
Reduce: `
|
||||
function(previous, current) {
|
||||
var count = 0;
|
||||
|
||||
for (index in current) {
|
||||
count += current[index];
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
`,
|
||||
}
|
||||
if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
|
||||
return kp, err
|
||||
}
|
||||
// Less than effecient, but cleanest place to put this
|
||||
for i := range kp {
|
||||
kp[i].Root = "ext" // for extension. Maps to /ext/
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
|
||||
// get a list of keywords and their frequency count
|
||||
func (h mongoHandle) GetKeywords() (kp []types.IdCount, err error) {
|
||||
job := &mgo.MapReduce{
|
||||
Map: `
|
||||
function() {
|
||||
if (!this.metadata.keywords) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (index in this.metadata.keywords) {
|
||||
emit(this.metadata.keywords[index], 1);
|
||||
}
|
||||
}
|
||||
`,
|
||||
Reduce: `
|
||||
function(previous, current) {
|
||||
var count = 0;
|
||||
|
||||
for (index in current) {
|
||||
count += current[index];
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
`,
|
||||
}
|
||||
if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
|
||||
return kp, err
|
||||
}
|
||||
// Less than effecient, but cleanest place to put this
|
||||
for i := range kp {
|
||||
kp[i].Root = "k" // for keyword. Maps to /k/
|
||||
}
|
||||
return kp, nil
|
||||
}
|
||||
26
server.go
26
server.go
|
|
@ -15,6 +15,7 @@ import (
|
|||
"github.com/vbatts/imgsrv/assets"
|
||||
"github.com/vbatts/imgsrv/config"
|
||||
"github.com/vbatts/imgsrv/dbutil"
|
||||
_ "github.com/vbatts/imgsrv/dbutil/mongo"
|
||||
"github.com/vbatts/imgsrv/hash"
|
||||
"github.com/vbatts/imgsrv/types"
|
||||
"github.com/vbatts/imgsrv/util"
|
||||
|
|
@ -24,24 +25,27 @@ var (
|
|||
defaultPageLimit int = 25
|
||||
maxBytes int64 = 1024 * 512
|
||||
serverConfig config.Config
|
||||
du dbutil.Util
|
||||
du dbutil.Handler
|
||||
)
|
||||
|
||||
/*
|
||||
Run as the file/image server
|
||||
*/
|
||||
// Run as the file/image server
|
||||
func runServer(c *config.Config) {
|
||||
serverConfig = *c
|
||||
|
||||
du = dbutil.Util{
|
||||
Seed: serverConfig.MongoHost,
|
||||
User: serverConfig.MongoUsername,
|
||||
Pass: serverConfig.MongoPassword,
|
||||
DbName: serverConfig.MongoDbName,
|
||||
du = dbutil.Handles["mongo"]
|
||||
duConfig := struct {
|
||||
Seed string
|
||||
User string
|
||||
Pass string
|
||||
DbName string
|
||||
}{
|
||||
serverConfig.MongoHost,
|
||||
serverConfig.MongoUsername,
|
||||
serverConfig.MongoPassword,
|
||||
serverConfig.MongoDbName,
|
||||
}
|
||||
|
||||
err := du.Init()
|
||||
if err != nil {
|
||||
if err := du.Init(duConfig); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer du.Close() // TODO this ought to catch a signal to cleanup
|
||||
|
|
|
|||
Loading…
Reference in a new issue