From efa8ac6511bd020e0b8deea4b5570b0d84260b91 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 7 Apr 2017 11:32:39 -0400 Subject: [PATCH] dbutil: abstract out mongo Handler Signed-off-by: Vincent Batts --- dbutil/dbutil.go | 273 ++++------------------------------------- dbutil/mongo/handle.go | 228 ++++++++++++++++++++++++++++++++++ server.go | 38 +++--- 3 files changed, 273 insertions(+), 266 deletions(-) create mode 100644 dbutil/mongo/handle.go diff --git a/dbutil/dbutil.go b/dbutil/dbutil.go index f8ae072..24220ba 100644 --- a/dbutil/dbutil.go +++ b/dbutil/dbutil.go @@ -1,258 +1,33 @@ package dbutil import ( - "github.com/vbatts/imgsrv/hash" "github.com/vbatts/imgsrv/types" "labix.org/v2/mgo" - "labix.org/v2/mgo/bson" - "strings" ) -const ( - DEFAULT_DB_NAME = "filesrv" -) +// Handles are all the register backing Handlers +var Handles map[string]Handler -type Util struct { - Seed string // mongo host seed to Dial into - User string // mongo credentials, if needed - Pass string // mongo credentials, if needed - DbName string // mongo database name, if needed - Session *mgo.Session - FileDb *mgo.Database - Gfs *mgo.GridFS -} - -func (u *Util) Init() error { - var err error - u.Session, err = mgo.Dial(u.Seed) - if err != nil { - return err - } - - if len(u.DbName) > 0 { - u.FileDb = u.Session.DB(u.DbName) - } else { - u.FileDb = u.Session.DB(DEFAULT_DB_NAME) - } - - if len(u.User) > 0 && len(u.Pass) > 0 { - err = u.FileDb.Login(u.User, u.Pass) - if err != nil { - return err - } - } - u.Gfs = u.FileDb.GridFS("fs") - return nil -} - -func (u Util) Close() { - u.Session.Close() -} - -/* -pass through for GridFs -*/ -func (u Util) Open(filename string) (file *mgo.GridFile, err error) { - return u.Gfs.Open(strings.ToLower(filename)) -} - -/* -pass through for GridFs -*/ -func (u Util) Create(filename string) (file *mgo.GridFile, err error) { - return u.Gfs.Create(strings.ToLower(filename)) -} - -/* -pass through for GridFs -*/ -func (u Util) Remove(filename string) (err error) { - return u.Gfs.Remove(strings.ToLower(filename)) -} - -/* -Find files by their MD5 checksum -*/ -func (u Util) FindFilesByMd5(md5 string) (files []types.File, err error) { - err = u.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files) - return files, err -} - -/* -match for file name -*/ -func (u Util) FindFilesByName(filename string) (files []types.File, err error) { - err = u.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files) - return files, err -} - -/* -Case-insensitive pattern match for file name -*/ -func (u Util) FindFilesByPatt(filename_pat string) (files []types.File, err error) { - err = u.Gfs.Find(bson.M{"filename": bson.M{"$regex": filename_pat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files) - return files, err -} - -/* -Case-insensitive pattern match for file name -*/ -func (u Util) FindFilesByKeyword(keyword string) (files []types.File, err error) { - err = u.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files) - return files, err -} - -/* -Get all the files. - -pass -1 for all files -*/ -func (u Util) GetFiles(limit int) (files []types.File, err error) { - //files = []types.File{} - if limit == -1 { - err = u.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files) - } else { - err = u.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files) - } - return files, err -} - -/* -Count the filename matches -*/ -func (u Util) CountFiles(filename string) (count int, err error) { - query := u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}) - return query.Count() -} - -/* -Get one file back, by searching by file name -*/ -func (u Util) GetFileByFilename(filename string) (this_file types.File, err error) { - err = u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&this_file) - if err != nil { - return this_file, err - } - return this_file, nil -} - -func (u Util) GetFileRandom() (this_file types.File, err error) { - r := hash.Rand64() - err = u.Gfs.Find(bson.M{"random": bson.M{"$gt": r}}).One(&this_file) - if err != nil { - return this_file, err - } - if len(this_file.Md5) == 0 { - err = u.Gfs.Find(bson.M{"random": bson.M{"$lt": r}}).One(&this_file) - } - if err != nil { - return this_file, err - } - return this_file, nil -} - -/* -Check whether this types.File filename is on Mongo -*/ -func (u Util) HasFileByFilename(filename string) (exists bool, err error) { - c, err := u.CountFiles(filename) - if err != nil { - return false, err - } - exists = (c > 0) - return exists, nil -} - -func (u Util) HasFileByMd5(md5 string) (exists bool, err error) { - c, err := u.Gfs.Find(bson.M{"md5": md5}).Count() - if err != nil { - return false, err - } - exists = (c > 0) - return exists, nil -} - -func (u Util) HasFileByKeyword(keyword string) (exists bool, err error) { - c, err := u.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count() - if err != nil { - return false, err - } - exists = (c > 0) - return exists, nil -} - -/* -get a list of file extensions and their frequency count -*/ -func (u Util) GetExtensions() (kp []types.IdCount, err error) { - job := &mgo.MapReduce{ - Map: ` - function() { - if (!this.filename) { - return; - } - - s = this.filename.split(".") - ext = s[s.length - 1] // get the last segment of the split - emit(ext,1); - } - `, - Reduce: ` - function(previous, current) { - var count = 0; - - for (index in current) { - count += current[index]; - } - - return count; - } - `, - } - if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil { - return kp, err - } - // Less than effecient, but cleanest place to put this - for i := range kp { - kp[i].Root = "ext" // for extension. Maps to /ext/ - } - return kp, nil -} - -/* -get a list of keywords and their frequency count -*/ -func (u Util) GetKeywords() (kp []types.IdCount, err error) { - job := &mgo.MapReduce{ - Map: ` - function() { - if (!this.metadata.keywords) { - return; - } - - for (index in this.metadata.keywords) { - emit(this.metadata.keywords[index], 1); - } - } - `, - Reduce: ` - function(previous, current) { - var count = 0; - - for (index in current) { - count += current[index]; - } - - return count; - } - `, - } - if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil { - return kp, err - } - // Less than effecient, but cleanest place to put this - for i := range kp { - kp[i].Root = "k" // for keyword. Maps to /k/ - } - return kp, nil +// Handler is the means of getting "files" from the backing database +type Handler interface { + Init(config interface{}) error + Close() error + + Open(filename string) (*mgo.GridFile, error) + Create(filename string) (*mgo.GridFile, error) + Remove(filename string) error + + //HasFileByMd5(md5 string) (exists bool, err error) + //HasFileByKeyword(keyword string) (exists bool, err error) + HasFileByFilename(filename string) (exists bool, err error) + FindFilesByKeyword(keyword string) (files []types.File, err error) + FindFilesByMd5(md5 string) (files []types.File, err error) + FindFilesByPatt(filenamePat string) (files []types.File, err error) + + CountFiles(filename string) (int, error) + + GetFiles(limit int) (files []types.File, err error) + GetFileByFilename(filename string) (types.File, error) + GetExtensions() (kp []types.IdCount, err error) + GetKeywords() (kp []types.IdCount, err error) } diff --git a/dbutil/mongo/handle.go b/dbutil/mongo/handle.go new file mode 100644 index 0000000..889efa8 --- /dev/null +++ b/dbutil/mongo/handle.go @@ -0,0 +1,228 @@ +package mongo + +import ( + "strings" + + "github.com/vbatts/imgsrv/dbutil" + "github.com/vbatts/imgsrv/types" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" +) + +func init() { + dbutil.Handles["mongo"] = &mongoHandle{} +} + +const defaultDbName = "filesrv" + +type dbConfig struct { + Seed string // mongo host seed to Dial into + User string // mongo credentials, if needed + Pass string // mongo credentials, if needed + DbName string // mongo database name, if needed +} + +type mongoHandle struct { + config dbConfig + Session *mgo.Session + FileDb *mgo.Database + Gfs *mgo.GridFS +} + +func (h *mongoHandle) Init(config interface{}) (err error) { + h.config = config.(dbConfig) + + h.Session, err = mgo.Dial(h.config.Seed) + if err != nil { + return err + } + + if len(h.config.DbName) > 0 { + h.FileDb = h.Session.DB(h.config.DbName) + } else { + h.FileDb = h.Session.DB(defaultDbName) + } + + if len(h.config.User) > 0 && len(h.config.Pass) > 0 { + err = h.FileDb.Login(h.config.User, h.config.Pass) + if err != nil { + return err + } + } + h.Gfs = h.FileDb.GridFS("fs") + return nil +} + +func (h mongoHandle) Close() error { + h.Session.Close() + return nil +} + +// pass through for GridFs +func (h mongoHandle) Open(filename string) (file *mgo.GridFile, err error) { + return h.Gfs.Open(strings.ToLower(filename)) +} + +// pass through for GridFs +func (h mongoHandle) Create(filename string) (file *mgo.GridFile, err error) { + return h.Gfs.Create(strings.ToLower(filename)) +} + +// pass through for GridFs +func (h mongoHandle) Remove(filename string) (err error) { + return h.Gfs.Remove(strings.ToLower(filename)) +} + +// Find files by their MD5 checksum +func (h mongoHandle) FindFilesByMd5(md5 string) (files []types.File, err error) { + err = h.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files) + return files, err +} + +// match for file name +// XXX this is not used +func (h mongoHandle) FindFilesByName(filename string) (files []types.File, err error) { + err = h.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files) + return files, err +} + +// Case-insensitive pattern match for file name +func (h mongoHandle) FindFilesByPatt(filenamePat string) (files []types.File, err error) { + err = h.Gfs.Find(bson.M{"filename": bson.M{"$regex": filenamePat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files) + return files, err +} + +// Case-insensitive pattern match for file name +func (h mongoHandle) FindFilesByKeyword(keyword string) (files []types.File, err error) { + err = h.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files) + return files, err +} + +// Get all the files. +// Pass -1 for all files. +func (h mongoHandle) GetFiles(limit int) (files []types.File, err error) { + //files = []types.File{} + if limit == -1 { + err = h.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files) + } else { + err = h.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files) + } + return files, err +} + +// Count the filename matches +func (h mongoHandle) CountFiles(filename string) (count int, err error) { + query := h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}) + return query.Count() +} + +// Get one file back, by searching by file name +func (h mongoHandle) GetFileByFilename(filename string) (thisFile types.File, err error) { + err = h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&thisFile) + if err != nil { + return thisFile, err + } + return thisFile, nil +} + +// Check whether this types.File filename is on Mongo +func (h mongoHandle) HasFileByFilename(filename string) (exists bool, err error) { + c, err := h.CountFiles(filename) + if err != nil { + return false, err + } + exists = (c > 0) + return exists, nil +} + +// XXX this is not used +func (h mongoHandle) HasFileByMd5(md5 string) (exists bool, err error) { + c, err := h.Gfs.Find(bson.M{"md5": md5}).Count() + if err != nil { + return false, err + } + exists = (c > 0) + return exists, nil +} + +// XXX this is not used +func (h mongoHandle) HasFileByKeyword(keyword string) (exists bool, err error) { + c, err := h.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count() + if err != nil { + return false, err + } + exists = (c > 0) + return exists, nil +} + +// get a list of file extensions and their frequency count +func (h mongoHandle) GetExtensions() (kp []types.IdCount, err error) { + job := &mgo.MapReduce{ + Map: ` + function() { + if (!this.filename) { + return; + } + + s = this.filename.split(".") + ext = s[s.length - 1] // get the last segment of the split + emit(ext,1); + } + `, + Reduce: ` + function(previous, current) { + var count = 0; + + for (index in current) { + count += current[index]; + } + + return count; + } + `, + } + if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil { + return kp, err + } + // Less than effecient, but cleanest place to put this + for i := range kp { + kp[i].Root = "ext" // for extension. Maps to /ext/ + } + return kp, nil +} + +// get a list of keywords and their frequency count +func (h mongoHandle) GetKeywords() (kp []types.IdCount, err error) { + job := &mgo.MapReduce{ + Map: ` + function() { + if (!this.metadata.keywords) { + return; + } + + for (index in this.metadata.keywords) { + emit(this.metadata.keywords[index], 1); + } + } + `, + Reduce: ` + function(previous, current) { + var count = 0; + + for (index in current) { + count += current[index]; + } + + return count; + } + `, + } + if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil { + return kp, err + } + // Less than effecient, but cleanest place to put this + for i := range kp { + kp[i].Root = "k" // for keyword. Maps to /k/ + } + return kp, nil +} diff --git a/server.go b/server.go index 2af758b..c411c42 100644 --- a/server.go +++ b/server.go @@ -15,6 +15,7 @@ import ( "github.com/vbatts/imgsrv/assets" "github.com/vbatts/imgsrv/config" "github.com/vbatts/imgsrv/dbutil" + _ "github.com/vbatts/imgsrv/dbutil/mongo" "github.com/vbatts/imgsrv/hash" "github.com/vbatts/imgsrv/types" "github.com/vbatts/imgsrv/util" @@ -24,24 +25,27 @@ var ( defaultPageLimit int = 25 maxBytes int64 = 1024 * 512 serverConfig config.Config - du dbutil.Util + du dbutil.Handler ) -/* -Run as the file/image server -*/ +// Run as the file/image server func runServer(c *config.Config) { serverConfig = *c - du = dbutil.Util{ - Seed: serverConfig.MongoHost, - User: serverConfig.MongoUsername, - Pass: serverConfig.MongoPassword, - DbName: serverConfig.MongoDbName, + du = dbutil.Handles["mongo"] + duConfig := struct { + Seed string + User string + Pass string + DbName string + }{ + serverConfig.MongoHost, + serverConfig.MongoUsername, + serverConfig.MongoPassword, + serverConfig.MongoDbName, } - err := du.Init() - if err != nil { + if err := du.Init(duConfig); err != nil { log.Fatal(err) } defer du.Close() // TODO this ought to catch a signal to cleanup @@ -225,10 +229,10 @@ func routeFilesPOST(w http.ResponseWriter, r *http.Request) { return } - // Keep it DRY? + // Keep it DRY? if r.MultipartForm != nil { routeUpload(w, r) - return + return } filename = r.FormValue("filename") @@ -784,10 +788,10 @@ func routeUpload(w http.ResponseWriter, r *http.Request) { n) } - if returnUrl { - fmt.Fprintf(w, "/v/%s", filename) - return - } + if returnUrl { + fmt.Fprintf(w, "/v/%s", filename) + return + } http.Redirect(w, r, fmt.Sprintf("/v/%s", filename), 302) } else { httplog.LogRequest(r, 404)