// Copyright 2014 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package main is an example Mananged VM app using the Google Cloud Storage API. package main //[START imports] import ( "bytes" "fmt" "io" "io/ioutil" "net/http" "strings" "cloud.google.com/go/storage" "golang.org/x/net/context" "google.golang.org/api/iterator" "google.golang.org/appengine" "google.golang.org/appengine/file" "google.golang.org/appengine/log" ) //[END imports] func main() { http.HandleFunc("/", handler) appengine.Main() } //[START bucket_struct] // demo struct holds information needed to run the various demo functions. type demo struct { client *storage.Client bucketName string bucket *storage.BucketHandle w io.Writer ctx context.Context // cleanUp is a list of filenames that need cleaning up at the end of the demo. cleanUp []string // failed indicates that one or more of the demo steps failed. failed bool } //[END bucket_struct] func (d *demo) errorf(format string, args ...interface{}) { d.failed = true fmt.Fprintln(d.w, fmt.Sprintf(format, args...)) log.Errorf(d.ctx, format, args...) } // handler is the main demo entry point that calls the GCS operations. func handler(w http.ResponseWriter, r *http.Request) { ctx := appengine.NewContext(r) if r.URL.Path != "/" { http.NotFound(w, r) return } //[START get_default_bucket] // Use `dev_appserver.py --default_gcs_bucket_name GCS_BUCKET_NAME` // when running locally. bucket, err := file.DefaultBucketName(ctx) if err != nil { log.Errorf(ctx, "failed to get default GCS bucket name: %v", err) } //[END get_default_bucket] client, err := storage.NewClient(ctx) if err != nil { log.Errorf(ctx, "failed to create client: %v", err) return } defer client.Close() w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(ctx)) fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket) buf := &bytes.Buffer{} d := &demo{ w: buf, ctx: ctx, client: client, bucket: client.Bucket(bucket), bucketName: bucket, } n := "demo-testfile-go" d.createFile(n) d.readFile(n) d.copyFile(n) d.statFile(n) d.createListFiles() d.listBucket() d.listBucketDirMode() d.defaultACL() d.putDefaultACLRule() d.deleteDefaultACLRule() d.bucketACL() d.putBucketACLRule() d.deleteBucketACLRule() d.acl(n) d.putACLRule(n) d.deleteACLRule(n) d.deleteFiles() if d.failed { w.WriteHeader(http.StatusInternalServerError) buf.WriteTo(w) fmt.Fprintf(w, "\nDemo failed.\n") } else { w.WriteHeader(http.StatusOK) buf.WriteTo(w) fmt.Fprintf(w, "\nDemo succeeded.\n") } } //[START write] // createFile creates a file in Google Cloud Storage. func (d *demo) createFile(fileName string) { fmt.Fprintf(d.w, "Creating file /%v/%v\n", d.bucket, fileName) wc := d.bucket.Object(fileName).NewWriter(d.ctx) wc.ContentType = "text/plain" wc.Metadata = map[string]string{ "x-goog-meta-foo": "foo", "x-goog-meta-bar": "bar", } d.cleanUp = append(d.cleanUp, fileName) if _, err := wc.Write([]byte("abcde\n")); err != nil { d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err) return } if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil { d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err) return } if err := wc.Close(); err != nil { d.errorf("createFile: unable to close bucket %q, file %q: %v", d.bucketName, fileName, err) return } } //[END write] //[START read] // readFile reads the named file in Google Cloud Storage. func (d *demo) readFile(fileName string) { io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n") rc, err := d.bucket.Object(fileName).NewReader(d.ctx) if err != nil { d.errorf("readFile: unable to open file from bucket %q, file %q: %v", d.bucketName, fileName, err) return } defer rc.Close() slurp, err := ioutil.ReadAll(rc) if err != nil { d.errorf("readFile: unable to read data from bucket %q, file %q: %v", d.bucketName, fileName, err) return } fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0]) if len(slurp) > 1024 { fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:]) } else { fmt.Fprintf(d.w, "%s\n", slurp) } } //[END read] //[START copy] // copyFile copies a file in Google Cloud Storage. func (d *demo) copyFile(fileName string) { copyName := fileName + "-copy" fmt.Fprintf(d.w, "Copying file /%v/%v to /%v/%v:\n", d.bucket, fileName, d.bucketName, copyName) obj, err := d.bucket.Object(copyName).CopierFrom(d.bucket.Object(fileName)).Run(d.ctx) if err != nil { d.errorf("copyFile: unable to copy /%v/%v to bucket %q, file %q: %v", d.bucketName, fileName, d.bucketName, copyName, err) return } d.cleanUp = append(d.cleanUp, copyName) d.dumpStats(obj) } //[END copy] func (d *demo) dumpStats(obj *storage.ObjectAttrs) { fmt.Fprintf(d.w, "(filename: /%v/%v, ", obj.Bucket, obj.Name) fmt.Fprintf(d.w, "ContentType: %q, ", obj.ContentType) fmt.Fprintf(d.w, "ACL: %#v, ", obj.ACL) fmt.Fprintf(d.w, "Owner: %v, ", obj.Owner) fmt.Fprintf(d.w, "ContentEncoding: %q, ", obj.ContentEncoding) fmt.Fprintf(d.w, "Size: %v, ", obj.Size) fmt.Fprintf(d.w, "MD5: %q, ", obj.MD5) fmt.Fprintf(d.w, "CRC32C: %q, ", obj.CRC32C) fmt.Fprintf(d.w, "Metadata: %#v, ", obj.Metadata) fmt.Fprintf(d.w, "MediaLink: %q, ", obj.MediaLink) fmt.Fprintf(d.w, "StorageClass: %q, ", obj.StorageClass) if !obj.Deleted.IsZero() { fmt.Fprintf(d.w, "Deleted: %v, ", obj.Deleted) } fmt.Fprintf(d.w, "Updated: %v)\n", obj.Updated) } //[START file_metadata] // statFile reads the stats of the named file in Google Cloud Storage. func (d *demo) statFile(fileName string) { io.WriteString(d.w, "\nFile stat:\n") obj, err := d.bucket.Object(fileName).Attrs(d.ctx) if err != nil { d.errorf("statFile: unable to stat file from bucket %q, file %q: %v", d.bucketName, fileName, err) return } d.dumpStats(obj) } //[END file_metadata] // createListFiles creates files that will be used by listBucket. func (d *demo) createListFiles() { io.WriteString(d.w, "\nCreating more files for listbucket...\n") for _, n := range []string{"foo1", "foo2", "bar", "bar/1", "bar/2", "boo/"} { d.createFile(n) } } //[START list_bucket] // listBucket lists the contents of a bucket in Google Cloud Storage. func (d *demo) listBucket() { io.WriteString(d.w, "\nListbucket result:\n") query := &storage.Query{Prefix: "foo"} it := d.bucket.Objects(d.ctx, query) for { obj, err := it.Next() if err == iterator.Done { break } if err != nil { d.errorf("listBucket: unable to list bucket %q: %v", d.bucketName, err) return } d.dumpStats(obj) } } //[END list_bucket] func (d *demo) listDir(name, indent string) { query := &storage.Query{Prefix: name, Delimiter: "/"} it := d.bucket.Objects(d.ctx, query) for { obj, err := it.Next() if err == iterator.Done { break } if err != nil { d.errorf("listBucketDirMode: unable to list bucket %q: %v", d.bucketName, err) return } if obj.Prefix == "" { fmt.Fprint(d.w, indent) d.dumpStats(obj) continue } fmt.Fprintf(d.w, "%v(directory: /%v/%v)\n", indent, d.bucketName, obj.Prefix) d.listDir(obj.Prefix, indent+" ") } } // listBucketDirMode lists the contents of a bucket in dir mode in Google Cloud Storage. func (d *demo) listBucketDirMode() { io.WriteString(d.w, "\nListbucket directory mode result:\n") d.listDir("b", "") } // dumpDefaultACL prints out the default object ACL for this bucket. func (d *demo) dumpDefaultACL() { acl, err := d.bucket.ACL().List(d.ctx) if err != nil { d.errorf("defaultACL: unable to list default object ACL for bucket %q: %v", d.bucketName, err) return } for _, v := range acl { fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role) } } // defaultACL displays the default object ACL for this bucket. func (d *demo) defaultACL() { io.WriteString(d.w, "\nDefault object ACL:\n") d.dumpDefaultACL() } // putDefaultACLRule adds the "allUsers" default object ACL rule for this bucket. func (d *demo) putDefaultACLRule() { io.WriteString(d.w, "\nPut Default object ACL Rule:\n") err := d.bucket.DefaultObjectACL().Set(d.ctx, storage.AllUsers, storage.RoleReader) if err != nil { d.errorf("putDefaultACLRule: unable to save default object ACL rule for bucket %q: %v", d.bucketName, err) return } d.dumpDefaultACL() } // deleteDefaultACLRule deleted the "allUsers" default object ACL rule for this bucket. func (d *demo) deleteDefaultACLRule() { io.WriteString(d.w, "\nDelete Default object ACL Rule:\n") err := d.bucket.DefaultObjectACL().Delete(d.ctx, storage.AllUsers) if err != nil { d.errorf("deleteDefaultACLRule: unable to delete default object ACL rule for bucket %q: %v", d.bucketName, err) return } d.dumpDefaultACL() } // dumpBucketACL prints out the bucket ACL. func (d *demo) dumpBucketACL() { acl, err := d.bucket.ACL().List(d.ctx) if err != nil { d.errorf("dumpBucketACL: unable to list bucket ACL for bucket %q: %v", d.bucketName, err) return } for _, v := range acl { fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role) } } // bucketACL displays the bucket ACL for this bucket. func (d *demo) bucketACL() { io.WriteString(d.w, "\nBucket ACL:\n") d.dumpBucketACL() } // putBucketACLRule adds the "allUsers" bucket ACL rule for this bucket. func (d *demo) putBucketACLRule() { io.WriteString(d.w, "\nPut Bucket ACL Rule:\n") err := d.bucket.ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader) if err != nil { d.errorf("putBucketACLRule: unable to save bucket ACL rule for bucket %q: %v", d.bucketName, err) return } d.dumpBucketACL() } // deleteBucketACLRule deleted the "allUsers" bucket ACL rule for this bucket. func (d *demo) deleteBucketACLRule() { io.WriteString(d.w, "\nDelete Bucket ACL Rule:\n") err := d.bucket.ACL().Delete(d.ctx, storage.AllUsers) if err != nil { d.errorf("deleteBucketACLRule: unable to delete bucket ACL rule for bucket %q: %v", d.bucketName, err) return } d.dumpBucketACL() } // dumpACL prints out the ACL of the named file. func (d *demo) dumpACL(fileName string) { acl, err := d.bucket.Object(fileName).ACL().List(d.ctx) if err != nil { d.errorf("dumpACL: unable to list file ACL for bucket %q, file %q: %v", d.bucketName, fileName, err) return } for _, v := range acl { fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role) } } // acl displays the ACL for the named file. func (d *demo) acl(fileName string) { fmt.Fprintf(d.w, "\nACL for file %v:\n", fileName) d.dumpACL(fileName) } // putACLRule adds the "allUsers" ACL rule for the named file. func (d *demo) putACLRule(fileName string) { fmt.Fprintf(d.w, "\nPut ACL rule for file %v:\n", fileName) err := d.bucket.Object(fileName).ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader) if err != nil { d.errorf("putACLRule: unable to save ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err) return } d.dumpACL(fileName) } // deleteACLRule deleted the "allUsers" ACL rule for the named file. func (d *demo) deleteACLRule(fileName string) { fmt.Fprintf(d.w, "\nDelete ACL rule for file %v:\n", fileName) err := d.bucket.Object(fileName).ACL().Delete(d.ctx, storage.AllUsers) if err != nil { d.errorf("deleteACLRule: unable to delete ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err) return } d.dumpACL(fileName) } //[START delete] // deleteFiles deletes all the temporary files from a bucket created by this demo. func (d *demo) deleteFiles() { io.WriteString(d.w, "\nDeleting files...\n") for _, v := range d.cleanUp { fmt.Fprintf(d.w, "Deleting file %v\n", v) if err := d.bucket.Object(v).Delete(d.ctx); err != nil { d.errorf("deleteFiles: unable to delete bucket %q, file %q: %v", d.bucketName, v, err) return } } } //[END delete]