Merge pull request #9 from vbatts/dbhandler

Dbhandler

dbutil/dbutil.go

@@ -1,258 +1,70 @@
 package dbutil
 
 import (
-	"github.com/vbatts/imgsrv/hash"
+	"io"
+
 	"github.com/vbatts/imgsrv/types"
-	"labix.org/v2/mgo"
-	"labix.org/v2/mgo/bson"
-	"strings"
 )
 
-const (
+// Handles are all the register backing Handlers
-	DEFAULT_DB_NAME = "filesrv"
+var Handles = map[string]Handler{}
-)
 
-type Util struct {
+// Handler is the means of getting "files" from the backing database
-	Seed    string // mongo host seed to Dial into
+type Handler interface {
-	User    string // mongo credentials, if needed
+	Init(config []byte, err error) error
-	Pass    string // mongo credentials, if needed
+	Close() error
-	DbName  string // mongo database name, if needed
+
-	Session *mgo.Session
+	Open(filename string) (File, error)
-	FileDb  *mgo.Database
+	Create(filename string) (File, error)
-	Gfs     *mgo.GridFS
+	Remove(filename string) error
+
+	//HasFileByMd5(md5 string) (exists bool, err error)
+	//HasFileByKeyword(keyword string) (exists bool, err error)
+	HasFileByFilename(filename string) (exists bool, err error)
+	FindFilesByKeyword(keyword string) (files []types.File, err error)
+	FindFilesByMd5(md5 string) (files []types.File, err error)
+	FindFilesByPatt(filenamePat string) (files []types.File, err error)
+
+	CountFiles(filename string) (int, error)
+
+	GetFiles(limit int) (files []types.File, err error)
+	GetFileByFilename(filename string) (types.File, error)
+	GetExtensions() (kp []types.IdCount, err error)
+	GetKeywords() (kp []types.IdCount, err error)
 }
 
-func (u *Util) Init() error {
+// File is what is stored and fetched from the backing database
-	var err error
+type File interface {
-	u.Session, err = mgo.Dial(u.Seed)
+	io.Reader
-	if err != nil {
+	io.Writer
-		return err
+	io.Closer
-	}
+	MetaDataer
-
-	if len(u.DbName) > 0 {
-		u.FileDb = u.Session.DB(u.DbName)
-	} else {
-		u.FileDb = u.Session.DB(DEFAULT_DB_NAME)
-	}
-
-	if len(u.User) > 0 && len(u.Pass) > 0 {
-		err = u.FileDb.Login(u.User, u.Pass)
-		if err != nil {
-			return err
-		}
-	}
-	u.Gfs = u.FileDb.GridFS("fs")
-	return nil
 }
 
-func (u Util) Close() {
+// MetaDataer allows set/get for optional metadata
-	u.Session.Close()
+type MetaDataer interface {
-}
+	/*
-
+		GetMeta unmarshals the optional "metadata" field associated with the file into
-/*
+		the result parameter. The meaning of keys under that field is user-defined. For
-pass through for GridFs
+		example:
-*/
+
-func (u Util) Open(filename string) (file *mgo.GridFile, err error) {
+			result := struct{ INode int }{}
-	return u.Gfs.Open(strings.ToLower(filename))
+			err = file.GetMeta(&result)
-}
+			if err != nil {
-
+				panic(err.String())
-/*
+			}
-pass through for GridFs
+			fmt.Printf("inode: %d\n", result.INode)
-*/
+	*/
-func (u Util) Create(filename string) (file *mgo.GridFile, err error) {
+	GetMeta(result interface{}) (err error)
-	return u.Gfs.Create(strings.ToLower(filename))
+	/*
-}
+		SetMeta changes the optional "metadata" field associated with the file.  The
-
+		meaning of keys under that field is user-defined. For example:
-/*
+
-pass through for GridFs
+			file.SetMeta(bson.M{"inode": inode})
-*/
+
-func (u Util) Remove(filename string) (err error) {
+		It is a runtime error to call this function when the file is not open for
-	return u.Gfs.Remove(strings.ToLower(filename))
+		writing.
-}
+
-
+	*/
-/*
+	SetMeta(metadata interface{})
-Find files by their MD5 checksum
-*/
-func (u Util) FindFilesByMd5(md5 string) (files []types.File, err error) {
-	err = u.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
-	return files, err
-}
-
-/*
-match for file name
-*/
-func (u Util) FindFilesByName(filename string) (files []types.File, err error) {
-	err = u.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
-	return files, err
-}
-
-/*
-Case-insensitive pattern match for file name
-*/
-func (u Util) FindFilesByPatt(filename_pat string) (files []types.File, err error) {
-	err = u.Gfs.Find(bson.M{"filename": bson.M{"$regex": filename_pat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
-	return files, err
-}
-
-/*
-Case-insensitive pattern match for file name
-*/
-func (u Util) FindFilesByKeyword(keyword string) (files []types.File, err error) {
-	err = u.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
-	return files, err
-}
-
-/*
-Get all the files.
-
-pass -1 for all files
-*/
-func (u Util) GetFiles(limit int) (files []types.File, err error) {
-	//files = []types.File{}
-	if limit == -1 {
-		err = u.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
-	} else {
-		err = u.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
-	}
-	return files, err
-}
-
-/*
-Count the filename matches
-*/
-func (u Util) CountFiles(filename string) (count int, err error) {
-	query := u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
-	return query.Count()
-}
-
-/*
-Get one file back, by searching by file name
-*/
-func (u Util) GetFileByFilename(filename string) (this_file types.File, err error) {
-	err = u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&this_file)
-	if err != nil {
-		return this_file, err
-	}
-	return this_file, nil
-}
-
-func (u Util) GetFileRandom() (this_file types.File, err error) {
-	r := hash.Rand64()
-	err = u.Gfs.Find(bson.M{"random": bson.M{"$gt": r}}).One(&this_file)
-	if err != nil {
-		return this_file, err
-	}
-	if len(this_file.Md5) == 0 {
-		err = u.Gfs.Find(bson.M{"random": bson.M{"$lt": r}}).One(&this_file)
-	}
-	if err != nil {
-		return this_file, err
-	}
-	return this_file, nil
-}
-
-/*
-Check whether this types.File filename is on Mongo
-*/
-func (u Util) HasFileByFilename(filename string) (exists bool, err error) {
-	c, err := u.CountFiles(filename)
-	if err != nil {
-		return false, err
-	}
-	exists = (c > 0)
-	return exists, nil
-}
-
-func (u Util) HasFileByMd5(md5 string) (exists bool, err error) {
-	c, err := u.Gfs.Find(bson.M{"md5": md5}).Count()
-	if err != nil {
-		return false, err
-	}
-	exists = (c > 0)
-	return exists, nil
-}
-
-func (u Util) HasFileByKeyword(keyword string) (exists bool, err error) {
-	c, err := u.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
-	if err != nil {
-		return false, err
-	}
-	exists = (c > 0)
-	return exists, nil
-}
-
-/*
-get a list of file extensions and their frequency count
-*/
-func (u Util) GetExtensions() (kp []types.IdCount, err error) {
-	job := &mgo.MapReduce{
-		Map: `
-    function() {
-        if (!this.filename) {
-          return;
-        }
-
-        s = this.filename.split(".")
-        ext = s[s.length - 1] // get the last segment of the split
-        emit(ext,1);
-    }
-    `,
-		Reduce: `
-    function(previous, current) {
-      var count = 0;
-
-      for (index in current) {
-        count += current[index];
-      }
-
-      return count;
-    }
-    `,
-	}
-	if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
-		return kp, err
-	}
-	// Less than effecient, but cleanest place to put this
-	for i := range kp {
-		kp[i].Root = "ext" // for extension. Maps to /ext/
-	}
-	return kp, nil
-}
-
-/*
-get a list of keywords and their frequency count
-*/
-func (u Util) GetKeywords() (kp []types.IdCount, err error) {
-	job := &mgo.MapReduce{
-		Map: `
-    function() {
-        if (!this.metadata.keywords) {
-          return;
-        }
-
-        for (index in this.metadata.keywords) {
-          emit(this.metadata.keywords[index], 1);
-        }
-    }
-    `,
-		Reduce: `
-    function(previous, current) {
-      var count = 0;
-
-      for (index in current) {
-        count += current[index];
-      }
-
-      return count;
-    }
-    `,
-	}
-	if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
-		return kp, err
-	}
-	// Less than effecient, but cleanest place to put this
-	for i := range kp {
-		kp[i].Root = "k" // for keyword. Maps to /k/
-	}
-	return kp, nil
 }

dbutil/mongo/handle.go

@@ -0,0 +1,236 @@
+package mongo
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/vbatts/imgsrv/dbutil"
+	"github.com/vbatts/imgsrv/types"
+	"labix.org/v2/mgo"
+	"labix.org/v2/mgo/bson"
+)
+
+func init() {
+	dbutil.Handles["mongo"] = &mongoHandle{}
+}
+
+const defaultDbName = "filesrv"
+
+type dbConfig struct {
+	Seed   string // mongo host seed to Dial into
+	User   string // mongo credentials, if needed
+	Pass   string // mongo credentials, if needed
+	DbName string // mongo database name, if needed
+}
+
+type mongoHandle struct {
+	config  dbConfig
+	Session *mgo.Session
+	FileDb  *mgo.Database
+	Gfs     *mgo.GridFS
+}
+
+func (h *mongoHandle) Init(config []byte, err error) error {
+	if err != nil {
+		return err
+	}
+
+	h.config = dbConfig{}
+	if err := json.Unmarshal(config, &h.config); err != nil {
+		return err
+	}
+
+	h.Session, err = mgo.Dial(h.config.Seed)
+	if err != nil {
+		return err
+	}
+
+	if len(h.config.DbName) > 0 {
+		h.FileDb = h.Session.DB(h.config.DbName)
+	} else {
+		h.FileDb = h.Session.DB(defaultDbName)
+	}
+
+	if len(h.config.User) > 0 && len(h.config.Pass) > 0 {
+		err = h.FileDb.Login(h.config.User, h.config.Pass)
+		if err != nil {
+			return err
+		}
+	}
+	h.Gfs = h.FileDb.GridFS("fs")
+	return nil
+}
+
+func (h mongoHandle) Close() error {
+	h.Session.Close()
+	return nil
+}
+
+// pass through for GridFs
+func (h mongoHandle) Open(filename string) (file dbutil.File, err error) {
+	return h.Gfs.Open(strings.ToLower(filename))
+}
+
+// pass through for GridFs
+func (h mongoHandle) Create(filename string) (file dbutil.File, err error) {
+	return h.Gfs.Create(strings.ToLower(filename))
+}
+
+// pass through for GridFs
+func (h mongoHandle) Remove(filename string) (err error) {
+	return h.Gfs.Remove(strings.ToLower(filename))
+}
+
+// Find files by their MD5 checksum
+func (h mongoHandle) FindFilesByMd5(md5 string) (files []types.File, err error) {
+	err = h.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
+	return files, err
+}
+
+// match for file name
+// XXX this is not used
+func (h mongoHandle) FindFilesByName(filename string) (files []types.File, err error) {
+	err = h.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
+	return files, err
+}
+
+// Case-insensitive pattern match for file name
+func (h mongoHandle) FindFilesByPatt(filenamePat string) (files []types.File, err error) {
+	err = h.Gfs.Find(bson.M{"filename": bson.M{"$regex": filenamePat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
+	return files, err
+}
+
+// Case-insensitive pattern match for file name
+func (h mongoHandle) FindFilesByKeyword(keyword string) (files []types.File, err error) {
+	err = h.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
+	return files, err
+}
+
+// Get all the files.
+// Pass -1 for all files.
+func (h mongoHandle) GetFiles(limit int) (files []types.File, err error) {
+	//files = []types.File{}
+	if limit == -1 {
+		err = h.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
+	} else {
+		err = h.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
+	}
+	return files, err
+}
+
+// Count the filename matches
+func (h mongoHandle) CountFiles(filename string) (count int, err error) {
+	query := h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
+	return query.Count()
+}
+
+// Get one file back, by searching by file name
+func (h mongoHandle) GetFileByFilename(filename string) (thisFile types.File, err error) {
+	err = h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&thisFile)
+	if err != nil {
+		return thisFile, err
+	}
+	return thisFile, nil
+}
+
+// Check whether this types.File filename is on Mongo
+func (h mongoHandle) HasFileByFilename(filename string) (exists bool, err error) {
+	c, err := h.CountFiles(filename)
+	if err != nil {
+		return false, err
+	}
+	exists = (c > 0)
+	return exists, nil
+}
+
+// XXX this is not used
+func (h mongoHandle) HasFileByMd5(md5 string) (exists bool, err error) {
+	c, err := h.Gfs.Find(bson.M{"md5": md5}).Count()
+	if err != nil {
+		return false, err
+	}
+	exists = (c > 0)
+	return exists, nil
+}
+
+// XXX this is not used
+func (h mongoHandle) HasFileByKeyword(keyword string) (exists bool, err error) {
+	c, err := h.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
+	if err != nil {
+		return false, err
+	}
+	exists = (c > 0)
+	return exists, nil
+}
+
+// get a list of file extensions and their frequency count
+func (h mongoHandle) GetExtensions() (kp []types.IdCount, err error) {
+	job := &mgo.MapReduce{
+		Map: `
+    function() {
+        if (!this.filename) {
+          return;
+        }
+
+        s = this.filename.split(".")
+        ext = s[s.length - 1] // get the last segment of the split
+        emit(ext,1);
+    }
+    `,
+		Reduce: `
+    function(previous, current) {
+      var count = 0;
+
+      for (index in current) {
+        count += current[index];
+      }
+
+      return count;
+    }
+    `,
+	}
+	if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
+		return kp, err
+	}
+	// Less than effecient, but cleanest place to put this
+	for i := range kp {
+		kp[i].Root = "ext" // for extension. Maps to /ext/
+	}
+	return kp, nil
+}
+
+// get a list of keywords and their frequency count
+func (h mongoHandle) GetKeywords() (kp []types.IdCount, err error) {
+	job := &mgo.MapReduce{
+		Map: `
+    function() {
+        if (!this.metadata.keywords) {
+          return;
+        }
+
+        for (index in this.metadata.keywords) {
+          emit(this.metadata.keywords[index], 1);
+        }
+    }
+    `,
+		Reduce: `
+    function(previous, current) {
+      var count = 0;
+
+      for (index in current) {
+        count += current[index];
+      }
+
+      return count;
+    }
+    `,
+	}
+	if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
+		return kp, err
+	}
+	// Less than effecient, but cleanest place to put this
+	for i := range kp {
+		kp[i].Root = "k" // for keyword. Maps to /k/
+	}
+	return kp, nil
+}

server.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"log"
@@ -15,6 +16,7 @@ import (
 	"github.com/vbatts/imgsrv/assets"
 	"github.com/vbatts/imgsrv/config"
 	"github.com/vbatts/imgsrv/dbutil"
+	_ "github.com/vbatts/imgsrv/dbutil/mongo"
 	"github.com/vbatts/imgsrv/hash"
 	"github.com/vbatts/imgsrv/types"
 	"github.com/vbatts/imgsrv/util"
@@ -24,24 +26,27 @@ var (
 	defaultPageLimit int   = 25
 	maxBytes         int64 = 1024 * 512
 	serverConfig     config.Config
-	du               dbutil.Util
+	du               dbutil.Handler
 )
 
-/*
+// Run as the file/image server
-Run as the file/image server
-*/
 func runServer(c *config.Config) {
 	serverConfig = *c
 
-	du = dbutil.Util{
+	du = dbutil.Handles["mongo"]
-		Seed:   serverConfig.MongoHost,
+	duConfig := struct {
-		User:   serverConfig.MongoUsername,
+		Seed   string
-		Pass:   serverConfig.MongoPassword,
+		User   string
-		DbName: serverConfig.MongoDbName,
+		Pass   string
+		DbName string
+	}{
+		serverConfig.MongoHost,
+		serverConfig.MongoUsername,
+		serverConfig.MongoPassword,
+		serverConfig.MongoDbName,
 	}
 
-	err := du.Init()
+	if err := du.Init(json.Marshal(duConfig)); err != nil {
-	if err != nil {
 		log.Fatal(err)
 	}
 	defer du.Close() // TODO this ought to catch a signal to cleanup
@@ -225,10 +230,10 @@ func routeFilesPOST(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-  // Keep it DRY?
+	// Keep it DRY?
 	if r.MultipartForm != nil {
 		routeUpload(w, r)
-    return
+		return
 	}
 
 	filename = r.FormValue("filename")
@@ -784,10 +789,10 @@ func routeUpload(w http.ResponseWriter, r *http.Request) {
 				n)
 		}
 
-    if returnUrl {
+		if returnUrl {
-		  fmt.Fprintf(w, "/v/%s", filename)
+			fmt.Fprintf(w, "/v/%s", filename)
-      return
+			return
-    }
+		}
 		http.Redirect(w, r, fmt.Sprintf("/v/%s", filename), 302)
 	} else {
 		httplog.LogRequest(r, 404)

types/types.go

@@ -15,37 +15,32 @@ type Info struct {
 }
 
 type File struct {
-	Metadata    Info ",omitempty"
+	Metadata   Info ",omitempty"
-	Md5         string
+	Md5        string
-	ChunkSize   int
+	ChunkSize  int
-	UploadDate  time.Time
+	UploadDate time.Time
-	Length      int64
+	Length     int64
-	Filename    string ",omitempty"
+	Filename   string ",omitempty"
-	ContentType string "contentType,omitempty"
 }
 
-func (f *File) SetContentType() {
+// ContentType guesses the mime-type by the file's extension
-	f.ContentType = mime.TypeByExtension(filepath.Ext(f.Filename))
+func (f *File) ContentType() string {
+	return mime.TypeByExtension(filepath.Ext(f.Filename))
 }
 
 func (f *File) IsImage() bool {
-	f.SetContentType()
+	return strings.HasPrefix(f.ContentType(), "image")
-	return strings.HasPrefix(f.ContentType, "image")
 }
 
 func (f *File) IsVideo() bool {
-	f.SetContentType()
+	return strings.HasPrefix(f.ContentType(), "video")
-	return strings.HasPrefix(f.ContentType, "video")
 }
 
 func (f *File) IsAudio() bool {
-	f.SetContentType()
+	return strings.HasPrefix(f.ContentType(), "audio")
-	return strings.HasPrefix(f.ContentType, "audio")
 }
 
-/*
+// IdCount structure used for collecting values for a tag cloud
-Structure used for collecting values from mongo for a tag cloud
-*/
 type IdCount struct {
 	Id    string "_id"
 	Value int