mirrors/imgsrv
1
0
Fork 0
mirror of https://github.com/vbatts/imgsrv.git synced 2024-12-24 23:06:31 +00:00
Code Releases Activity

Merge pull request #9 from vbatts/dbhandler

Browse source 
Dbhandler
This commit is contained in:
Vincent Batts 2017-04-07 13:30:11 -04:00 committed by GitHub
commit 51545d2e46
4 changed files with 329 additions and 281 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

304
dbutil/dbutil.go
View file

@ -1,258 +1,70 @@
package dbutil
import (
"github.com/vbatts/imgsrv/hash"
"io"
"github.com/vbatts/imgsrv/types"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"strings"
)
const (
DEFAULT_DB_NAME = "filesrv"
)
// Handles are all the register backing Handlers
var Handles = map[string]Handler{}
type Util struct {
Seed string // mongo host seed to Dial into
User string // mongo credentials, if needed
Pass string // mongo credentials, if needed
DbName string // mongo database name, if needed
Session *mgo.Session
FileDb *mgo.Database
Gfs *mgo.GridFS
// Handler is the means of getting "files" from the backing database
type Handler interface {
Init(config []byte, err error) error
Close() error
Open(filename string) (File, error)
Create(filename string) (File, error)
Remove(filename string) error
//HasFileByMd5(md5 string) (exists bool, err error)
//HasFileByKeyword(keyword string) (exists bool, err error)
HasFileByFilename(filename string) (exists bool, err error)
FindFilesByKeyword(keyword string) (files []types.File, err error)
FindFilesByMd5(md5 string) (files []types.File, err error)
FindFilesByPatt(filenamePat string) (files []types.File, err error)
CountFiles(filename string) (int, error)
GetFiles(limit int) (files []types.File, err error)
GetFileByFilename(filename string) (types.File, error)
GetExtensions() (kp []types.IdCount, err error)
GetKeywords() (kp []types.IdCount, err error)
}
func (u *Util) Init() error {
var err error
u.Session, err = mgo.Dial(u.Seed)
if err != nil {
return err
}
if len(u.DbName) > 0 {
u.FileDb = u.Session.DB(u.DbName)
} else {
u.FileDb = u.Session.DB(DEFAULT_DB_NAME)
}
if len(u.User) > 0 && len(u.Pass) > 0 {
err = u.FileDb.Login(u.User, u.Pass)
if err != nil {
return err
}
}
u.Gfs = u.FileDb.GridFS("fs")
return nil
// File is what is stored and fetched from the backing database
type File interface {
io.Reader
io.Writer
io.Closer
MetaDataer
}
func (u Util) Close() {
u.Session.Close()
}
/*
pass through for GridFs
*/
func (u Util) Open(filename string) (file *mgo.GridFile, err error) {
return u.Gfs.Open(strings.ToLower(filename))
}
/*
pass through for GridFs
*/
func (u Util) Create(filename string) (file *mgo.GridFile, err error) {
return u.Gfs.Create(strings.ToLower(filename))
}
/*
pass through for GridFs
*/
func (u Util) Remove(filename string) (err error) {
return u.Gfs.Remove(strings.ToLower(filename))
}
/*
Find files by their MD5 checksum
*/
func (u Util) FindFilesByMd5(md5 string) (files []types.File, err error) {
err = u.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
return files, err
}
/*
match for file name
*/
func (u Util) FindFilesByName(filename string) (files []types.File, err error) {
err = u.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
return files, err
}
/*
Case-insensitive pattern match for file name
*/
func (u Util) FindFilesByPatt(filename_pat string) (files []types.File, err error) {
err = u.Gfs.Find(bson.M{"filename": bson.M{"$regex": filename_pat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
return files, err
}
/*
Case-insensitive pattern match for file name
*/
func (u Util) FindFilesByKeyword(keyword string) (files []types.File, err error) {
err = u.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
return files, err
}
/*
Get all the files.
pass -1 for all files
*/
func (u Util) GetFiles(limit int) (files []types.File, err error) {
//files = []types.File{}
if limit == -1 {
err = u.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
} else {
err = u.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
}
return files, err
}
/*
Count the filename matches
*/
func (u Util) CountFiles(filename string) (count int, err error) {
query := u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
return query.Count()
}
/*
Get one file back, by searching by file name
*/
func (u Util) GetFileByFilename(filename string) (this_file types.File, err error) {
err = u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&this_file)
if err != nil {
return this_file, err
}
return this_file, nil
}
func (u Util) GetFileRandom() (this_file types.File, err error) {
r := hash.Rand64()
err = u.Gfs.Find(bson.M{"random": bson.M{"$gt": r}}).One(&this_file)
if err != nil {
return this_file, err
}
if len(this_file.Md5) == 0 {
err = u.Gfs.Find(bson.M{"random": bson.M{"$lt": r}}).One(&this_file)
}
if err != nil {
return this_file, err
}
return this_file, nil
}
/*
Check whether this types.File filename is on Mongo
*/
func (u Util) HasFileByFilename(filename string) (exists bool, err error) {
c, err := u.CountFiles(filename)
if err != nil {
return false, err
}
exists = (c > 0)
return exists, nil
}
func (u Util) HasFileByMd5(md5 string) (exists bool, err error) {
c, err := u.Gfs.Find(bson.M{"md5": md5}).Count()
if err != nil {
return false, err
}
exists = (c > 0)
return exists, nil
}
func (u Util) HasFileByKeyword(keyword string) (exists bool, err error) {
c, err := u.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
if err != nil {
return false, err
}
exists = (c > 0)
return exists, nil
}
/*
get a list of file extensions and their frequency count
*/
func (u Util) GetExtensions() (kp []types.IdCount, err error) {
job := &mgo.MapReduce{
Map: `
function() {
if (!this.filename) {
return;
}
s = this.filename.split(".")
ext = s[s.length - 1] // get the last segment of the split
emit(ext,1);
}
`,
Reduce: `
function(previous, current) {
var count = 0;
for (index in current) {
count += current[index];
}
return count;
}
`,
}
if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
return kp, err
}
// Less than effecient, but cleanest place to put this
for i := range kp {
kp[i].Root = "ext" // for extension. Maps to /ext/
}
return kp, nil
}
/*
get a list of keywords and their frequency count
*/
func (u Util) GetKeywords() (kp []types.IdCount, err error) {
job := &mgo.MapReduce{
Map: `
function() {
if (!this.metadata.keywords) {
return;
}
for (index in this.metadata.keywords) {
emit(this.metadata.keywords[index], 1);
}
}
`,
Reduce: `
function(previous, current) {
var count = 0;
for (index in current) {
count += current[index];
}
return count;
}
`,
}
if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
return kp, err
}
// Less than effecient, but cleanest place to put this
for i := range kp {
kp[i].Root = "k" // for keyword. Maps to /k/
}
return kp, nil
// MetaDataer allows set/get for optional metadata
type MetaDataer interface {
/*
GetMeta unmarshals the optional "metadata" field associated with the file into
the result parameter. The meaning of keys under that field is user-defined. For
example:
result := struct{ INode int }{}
err = file.GetMeta(&result)
if err != nil {
panic(err.String())
}
fmt.Printf("inode: %d\n", result.INode)
*/
GetMeta(result interface{}) (err error)
/*
SetMeta changes the optional "metadata" field associated with the file. The
meaning of keys under that field is user-defined. For example:
file.SetMeta(bson.M{"inode": inode})
It is a runtime error to call this function when the file is not open for
writing.
*/
SetMeta(metadata interface{})
}

236
dbutil/mongo/handle.go Normal file
View file

@ -0,0 +1,236 @@
package mongo
import (
"encoding/json"
"strings"
"github.com/vbatts/imgsrv/dbutil"
"github.com/vbatts/imgsrv/types"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
)
func init() {
dbutil.Handles["mongo"] = &mongoHandle{}
}
const defaultDbName = "filesrv"
type dbConfig struct {
Seed string // mongo host seed to Dial into
User string // mongo credentials, if needed
Pass string // mongo credentials, if needed
DbName string // mongo database name, if needed
}
type mongoHandle struct {
config dbConfig
Session *mgo.Session
FileDb *mgo.Database
Gfs *mgo.GridFS
}
func (h *mongoHandle) Init(config []byte, err error) error {
if err != nil {
return err
}
h.config = dbConfig{}
if err := json.Unmarshal(config, &h.config); err != nil {
return err
}
h.Session, err = mgo.Dial(h.config.Seed)
if err != nil {
return err
}
if len(h.config.DbName) > 0 {
h.FileDb = h.Session.DB(h.config.DbName)
} else {
h.FileDb = h.Session.DB(defaultDbName)
}
if len(h.config.User) > 0 && len(h.config.Pass) > 0 {
err = h.FileDb.Login(h.config.User, h.config.Pass)
if err != nil {
return err
}
}
h.Gfs = h.FileDb.GridFS("fs")
return nil
}
func (h mongoHandle) Close() error {
h.Session.Close()
return nil
}
// pass through for GridFs
func (h mongoHandle) Open(filename string) (file dbutil.File, err error) {
return h.Gfs.Open(strings.ToLower(filename))
}
// pass through for GridFs
func (h mongoHandle) Create(filename string) (file dbutil.File, err error) {
return h.Gfs.Create(strings.ToLower(filename))
}
// pass through for GridFs
func (h mongoHandle) Remove(filename string) (err error) {
return h.Gfs.Remove(strings.ToLower(filename))
}
// Find files by their MD5 checksum
func (h mongoHandle) FindFilesByMd5(md5 string) (files []types.File, err error) {
err = h.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
return files, err
}
// match for file name
// XXX this is not used
func (h mongoHandle) FindFilesByName(filename string) (files []types.File, err error) {
err = h.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
return files, err
}
// Case-insensitive pattern match for file name
func (h mongoHandle) FindFilesByPatt(filenamePat string) (files []types.File, err error) {
err = h.Gfs.Find(bson.M{"filename": bson.M{"$regex": filenamePat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
return files, err
}
// Case-insensitive pattern match for file name
func (h mongoHandle) FindFilesByKeyword(keyword string) (files []types.File, err error) {
err = h.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
return files, err
}
// Get all the files.
// Pass -1 for all files.
func (h mongoHandle) GetFiles(limit int) (files []types.File, err error) {
//files = []types.File{}
if limit == -1 {
err = h.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
} else {
err = h.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
}
return files, err
}
// Count the filename matches
func (h mongoHandle) CountFiles(filename string) (count int, err error) {
query := h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
return query.Count()
}
// Get one file back, by searching by file name
func (h mongoHandle) GetFileByFilename(filename string) (thisFile types.File, err error) {
err = h.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&thisFile)
if err != nil {
return thisFile, err
}
return thisFile, nil
}
// Check whether this types.File filename is on Mongo
func (h mongoHandle) HasFileByFilename(filename string) (exists bool, err error) {
c, err := h.CountFiles(filename)
if err != nil {
return false, err
}
exists = (c > 0)
return exists, nil
}
// XXX this is not used
func (h mongoHandle) HasFileByMd5(md5 string) (exists bool, err error) {
c, err := h.Gfs.Find(bson.M{"md5": md5}).Count()
if err != nil {
return false, err
}
exists = (c > 0)
return exists, nil
}
// XXX this is not used
func (h mongoHandle) HasFileByKeyword(keyword string) (exists bool, err error) {
c, err := h.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
if err != nil {
return false, err
}
exists = (c > 0)
return exists, nil
}
// get a list of file extensions and their frequency count
func (h mongoHandle) GetExtensions() (kp []types.IdCount, err error) {
job := &mgo.MapReduce{
Map: `
function() {
if (!this.filename) {
return;
}
s = this.filename.split(".")
ext = s[s.length - 1] // get the last segment of the split
emit(ext,1);
}
`,
Reduce: `
function(previous, current) {
var count = 0;
for (index in current) {
count += current[index];
}
return count;
}
`,
}
if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
return kp, err
}
// Less than effecient, but cleanest place to put this
for i := range kp {
kp[i].Root = "ext" // for extension. Maps to /ext/
}
return kp, nil
}
// get a list of keywords and their frequency count
func (h mongoHandle) GetKeywords() (kp []types.IdCount, err error) {
job := &mgo.MapReduce{
Map: `
function() {
if (!this.metadata.keywords) {
return;
}
for (index in this.metadata.keywords) {
emit(this.metadata.keywords[index], 1);
}
}
`,
Reduce: `
function(previous, current) {
var count = 0;
for (index in current) {
count += current[index];
}
return count;
}
`,
}
if _, err := h.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
return kp, err
}
// Less than effecient, but cleanest place to put this
for i := range kp {
kp[i].Root = "k" // for keyword. Maps to /k/
}
return kp, nil
}

39
server.go
View file

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"io"
"log"
@ -15,6 +16,7 @@ import (
"github.com/vbatts/imgsrv/assets"
"github.com/vbatts/imgsrv/config"
"github.com/vbatts/imgsrv/dbutil"
_ "github.com/vbatts/imgsrv/dbutil/mongo"
"github.com/vbatts/imgsrv/hash"
"github.com/vbatts/imgsrv/types"
"github.com/vbatts/imgsrv/util"
@ -24,24 +26,27 @@ var (
defaultPageLimit int = 25
maxBytes int64 = 1024 * 512
serverConfig config.Config
du dbutil.Util
du dbutil.Handler
)
/*
Run as the file/image server
*/
// Run as the file/image server
func runServer(c *config.Config) {
serverConfig = *c
du = dbutil.Util{
Seed: serverConfig.MongoHost,
User: serverConfig.MongoUsername,
Pass: serverConfig.MongoPassword,
DbName: serverConfig.MongoDbName,
du = dbutil.Handles["mongo"]
duConfig := struct {
Seed string
User string
Pass string
DbName string
}{
serverConfig.MongoHost,
serverConfig.MongoUsername,
serverConfig.MongoPassword,
serverConfig.MongoDbName,
}
err := du.Init()
if err != nil {
if err := du.Init(json.Marshal(duConfig)); err != nil {
log.Fatal(err)
}
defer du.Close() // TODO this ought to catch a signal to cleanup
@ -225,10 +230,10 @@ func routeFilesPOST(w http.ResponseWriter, r *http.Request) {
return
}
// Keep it DRY?
// Keep it DRY?
if r.MultipartForm != nil {
routeUpload(w, r)
return
return
}
filename = r.FormValue("filename")
@ -784,10 +789,10 @@ func routeUpload(w http.ResponseWriter, r *http.Request) {
n)
}
if returnUrl {
fmt.Fprintf(w, "/v/%s", filename)
return
}
if returnUrl {
fmt.Fprintf(w, "/v/%s", filename)
return
}
http.Redirect(w, r, fmt.Sprintf("/v/%s", filename), 302)
} else {
httplog.LogRequest(r, 404)

31
types/types.go
View file

@ -15,37 +15,32 @@ type Info struct {
}
type File struct {
Metadata Info ",omitempty"
Md5 string
ChunkSize int
UploadDate time.Time
Length int64
Filename string ",omitempty"
ContentType string "contentType,omitempty"
Metadata Info ",omitempty"
Md5 string
ChunkSize int
UploadDate time.Time
Length int64
Filename string ",omitempty"
}
func (f *File) SetContentType() {
f.ContentType = mime.TypeByExtension(filepath.Ext(f.Filename))
// ContentType guesses the mime-type by the file's extension
func (f *File) ContentType() string {
return mime.TypeByExtension(filepath.Ext(f.Filename))
}
func (f *File) IsImage() bool {
f.SetContentType()
return strings.HasPrefix(f.ContentType, "image")
return strings.HasPrefix(f.ContentType(), "image")
}
func (f *File) IsVideo() bool {
f.SetContentType()
return strings.HasPrefix(f.ContentType, "video")
return strings.HasPrefix(f.ContentType(), "video")
}
func (f *File) IsAudio() bool {
f.SetContentType()
return strings.HasPrefix(f.ContentType, "audio")
return strings.HasPrefix(f.ContentType(), "audio")
}
/*
Structure used for collecting values from mongo for a tag cloud
*/
// IdCount structure used for collecting values for a tag cloud
type IdCount struct {
Id string "_id"
Value int