From a903880f82e5f661942205c02d11c2d19ceab49c Mon Sep 17 00:00:00 2001 From: Hayden <64056131+hay-kot@users.noreply.github.com> Date: Tue, 6 Sep 2022 11:15:07 -0800 Subject: [PATCH] initialize CSV Importer --- backend/app/api/docs/docs.go | 30 ++++ backend/app/api/docs/swagger.json | 30 ++++ backend/app/api/docs/swagger.yaml | 18 +++ backend/app/api/routes.go | 1 + backend/app/api/v1/v1_ctrl_items.go | 43 ++++++ backend/internal/repo/repo_items.go | 4 - backend/internal/repo/repo_locations.go | 4 + backend/internal/services/main_test.go | 70 +++++++++ backend/internal/services/service_items.go | 137 +++++++++++++++++- .../internal/services/service_items_csv.go | 97 +++++++++++++ .../services/service_items_csv_test.go | 87 +++++++++++ .../internal/services/service_items_test.go | 84 +++++++++++ 12 files changed, 600 insertions(+), 5 deletions(-) create mode 100644 backend/internal/services/main_test.go create mode 100644 backend/internal/services/service_items_csv.go create mode 100644 backend/internal/services/service_items_csv_test.go create mode 100644 backend/internal/services/service_items_test.go diff --git a/backend/app/api/docs/docs.go b/backend/app/api/docs/docs.go index 4e23987..eab5eaf 100644 --- a/backend/app/api/docs/docs.go +++ b/backend/app/api/docs/docs.go @@ -93,6 +93,36 @@ const docTemplate = `{ } } }, + "/v1/items/import": { + "post": { + "security": [ + { + "Bearer": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "Items" + ], + "summary": "imports items into the database", + "parameters": [ + { + "type": "file", + "description": "Image to upload", + "name": "csv", + "in": "formData", + "required": true + } + ], + "responses": { + "204": { + "description": "" + } + } + } + }, "/v1/items/{id}": { "get": { "security": [ diff --git a/backend/app/api/docs/swagger.json b/backend/app/api/docs/swagger.json index 7fc1b9f..bba2016 100644 --- a/backend/app/api/docs/swagger.json +++ b/backend/app/api/docs/swagger.json @@ -85,6 +85,36 @@ } } }, + "/v1/items/import": { + "post": { + "security": [ + { + "Bearer": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "Items" + ], + "summary": "imports items into the database", + "parameters": [ + { + "type": "file", + "description": "Image to upload", + "name": "csv", + "in": "formData", + "required": true + } + ], + "responses": { + "204": { + "description": "" + } + } + } + }, "/v1/items/{id}": { "get": { "security": [ diff --git a/backend/app/api/docs/swagger.yaml b/backend/app/api/docs/swagger.yaml index 23d526e..eee35b7 100644 --- a/backend/app/api/docs/swagger.yaml +++ b/backend/app/api/docs/swagger.yaml @@ -677,6 +677,24 @@ paths: summary: updates a item tags: - Items + /v1/items/import: + post: + parameters: + - description: Image to upload + in: formData + name: csv + required: true + type: file + produces: + - application/json + responses: + "204": + description: "" + security: + - Bearer: [] + summary: imports items into the database + tags: + - Items /v1/labels: get: produces: diff --git a/backend/app/api/routes.go b/backend/app/api/routes.go index 1ff79f0..1ee0a74 100644 --- a/backend/app/api/routes.go +++ b/backend/app/api/routes.go @@ -71,6 +71,7 @@ func (a *app) newRouter(repos *repo.AllRepos) *chi.Mux { r.Delete(v1Base("/labels/{id}"), v1Ctrl.HandleLabelDelete()) r.Get(v1Base("/items"), v1Ctrl.HandleItemsGetAll()) + r.Post(v1Base("/items/import"), v1Ctrl.HandleItemsImport()) r.Post(v1Base("/items"), v1Ctrl.HandleItemsCreate()) r.Get(v1Base("/items/{id}"), v1Ctrl.HandleItemGet()) r.Put(v1Base("/items/{id}"), v1Ctrl.HandleItemUpdate()) diff --git a/backend/app/api/v1/v1_ctrl_items.go b/backend/app/api/v1/v1_ctrl_items.go index c2ba923..9233e95 100644 --- a/backend/app/api/v1/v1_ctrl_items.go +++ b/backend/app/api/v1/v1_ctrl_items.go @@ -1,6 +1,7 @@ package v1 import ( + "encoding/csv" "net/http" "github.com/hay-kot/content/backend/internal/services" @@ -140,3 +141,45 @@ func (ctrl *V1Controller) HandleItemUpdate() http.HandlerFunc { server.Respond(w, http.StatusOK, result) } } + +// HandleItemsImport godocs +// @Summary imports items into the database +// @Tags Items +// @Produce json +// @Success 204 +// @Param csv formData file true "Image to upload" +// @Router /v1/items/import [Post] +// @Security Bearer +func (ctrl *V1Controller) HandleItemsImport() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + // Max upload size of 10 MB - TODO: Set via config + r.ParseMultipartForm(10 << 20) + + file, _, err := r.FormFile("csv") + if err != nil { + log.Err(err).Msg("failed to get file from form") + server.RespondServerError(w) + return + } + + reader := csv.NewReader(file) + data, err := reader.ReadAll() + if err != nil { + log.Err(err).Msg("failed to read csv") + server.RespondServerError(w) + return + } + + user := services.UseUserCtx(r.Context()) + + err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data) + if err != nil { + log.Err(err).Msg("failed to import items") + server.RespondServerError(w) + return + } + + server.Respond(w, http.StatusNoContent, nil) + } +} diff --git a/backend/internal/repo/repo_items.go b/backend/internal/repo/repo_items.go index 09a2101..f1f1ace 100644 --- a/backend/internal/repo/repo_items.go +++ b/backend/internal/repo/repo_items.go @@ -72,10 +72,6 @@ func (e *ItemsRepository) Update(ctx context.Context, data types.ItemUpdate) (*e SetSoldNotes(data.SoldNotes). SetNotes(data.Notes) - if data.LabelIDs != nil && len(data.LabelIDs) > 0 { - q.AddLabelIDs(data.LabelIDs...) - } - err := q.Exec(ctx) if err != nil { diff --git a/backend/internal/repo/repo_locations.go b/backend/internal/repo/repo_locations.go index b23b839..a1ce055 100644 --- a/backend/internal/repo/repo_locations.go +++ b/backend/internal/repo/repo_locations.go @@ -78,6 +78,10 @@ func (r *LocationRepository) Create(ctx context.Context, groupdId uuid.UUID, dat SetGroupID(groupdId). Save(ctx) + if err != nil { + return nil, err + } + location.Edges.Group = &ent.Group{ID: groupdId} // bootstrap group ID return location, err } diff --git a/backend/internal/services/main_test.go b/backend/internal/services/main_test.go new file mode 100644 index 0000000..9f65278 --- /dev/null +++ b/backend/internal/services/main_test.go @@ -0,0 +1,70 @@ +package services + +import ( + "context" + "log" + "math/rand" + "os" + "testing" + "time" + + "github.com/hay-kot/content/backend/ent" + "github.com/hay-kot/content/backend/internal/repo" + "github.com/hay-kot/content/backend/internal/types" + "github.com/hay-kot/content/backend/pkgs/faker" + _ "github.com/mattn/go-sqlite3" +) + +var ( + fk = faker.NewFaker() + + tClient *ent.Client + tRepos *repo.AllRepos + tUser *ent.User + tGroup *ent.Group +) + +func bootstrap() { + var ( + err error + ctx = context.Background() + ) + + tGroup, err = tRepos.Groups.Create(ctx, "test-group") + if err != nil { + log.Fatal(err) + } + + tUser, err = tRepos.Users.Create(ctx, types.UserCreate{ + Name: fk.RandomString(10), + Email: fk.RandomEmail(), + Password: fk.RandomString(10), + IsSuperuser: fk.RandomBool(), + GroupID: tGroup.ID, + }) + if err != nil { + log.Fatal(err) + } +} + +func TestMain(m *testing.M) { + rand.Seed(int64(time.Now().Unix())) + + client, err := ent.Open("sqlite3", "file:ent?mode=memory&cache=shared&_fk=1") + if err != nil { + log.Fatalf("failed opening connection to sqlite: %v", err) + } + + err = client.Schema.Create(context.Background()) + if err != nil { + log.Fatalf("failed creating schema resources: %v", err) + } + + tClient = client + tRepos = repo.EntAllRepos(tClient) + defer client.Close() + + bootstrap() + + os.Exit(m.Run()) +} diff --git a/backend/internal/services/service_items.go b/backend/internal/services/service_items.go index 3ec277d..00b54c9 100644 --- a/backend/internal/services/service_items.go +++ b/backend/internal/services/service_items.go @@ -2,11 +2,13 @@ package services import ( "context" + "fmt" "github.com/google/uuid" "github.com/hay-kot/content/backend/internal/repo" "github.com/hay-kot/content/backend/internal/services/mappers" "github.com/hay-kot/content/backend/internal/types" + "github.com/rs/zerolog/log" ) type ItemService struct { @@ -14,8 +16,18 @@ type ItemService struct { } func (svc *ItemService) GetOne(ctx context.Context, gid uuid.UUID, id uuid.UUID) (*types.ItemOut, error) { - panic("implement me") + result, err := svc.repo.Items.GetOne(ctx, id) + if err != nil { + return nil, err + } + + if result.Edges.Group.ID != gid { + return nil, ErrNotOwner + } + + return mappers.ToItemOut(result), nil } + func (svc *ItemService) GetAll(ctx context.Context, gid uuid.UUID) ([]*types.ItemSummary, error) { items, err := svc.repo.Items.GetAll(ctx, gid) if err != nil { @@ -43,3 +55,126 @@ func (svc *ItemService) Delete(ctx context.Context, gid uuid.UUID, id uuid.UUID) func (svc *ItemService) Update(ctx context.Context, gid uuid.UUID, data types.ItemUpdate) (*types.ItemOut, error) { panic("implement me") } + +func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) error { + loaded := []csvRow{} + + // Skip first row + for _, row := range data[1:] { + // Skip empty rows + if len(row) == 0 { + continue + } + if len(row) != 14 { + return ErrInvalidCsv + } + + r := newCsvRow(row) + loaded = append(loaded, r) + } + + // Bootstrap the locations and labels so we can reuse the created IDs for the items + locations := map[string]uuid.UUID{} + existingLocation, err := svc.repo.Locations.GetAll(ctx, gid) + if err != nil { + return err + } + for _, loc := range existingLocation { + locations[loc.Name] = loc.ID + } + + labels := map[string]uuid.UUID{} + existingLabels, err := svc.repo.Labels.GetAll(ctx, gid) + if err != nil { + return err + } + for _, label := range existingLabels { + labels[label.Name] = label.ID + } + + for _, row := range loaded { + + // Locations + if _, ok := locations[row.Location]; ok { + continue + } + + fmt.Println("Creating Location: ", row.Location) + + result, err := svc.repo.Locations.Create(ctx, gid, types.LocationCreate{ + Name: row.Location, + Description: "", + }) + if err != nil { + return err + } + locations[row.Location] = result.ID + + // Labels + + for _, label := range row.getLabels() { + if _, ok := labels[label]; ok { + continue + } + result, err := svc.repo.Labels.Create(ctx, gid, types.LabelCreate{ + Name: label, + Description: "", + }) + if err != nil { + return err + } + labels[label] = result.ID + } + } + + // Create the items + for _, row := range loaded { + locationID := locations[row.Location] + labelIDs := []uuid.UUID{} + for _, label := range row.getLabels() { + labelIDs = append(labelIDs, labels[label]) + } + + log.Info(). + Str("name", row.Name). + Str("location", row.Location). + Strs("labels", row.getLabels()). + Str("locationId", locationID.String()). + Msgf("Creating Item: %s", row.Name) + + result, err := svc.repo.Items.Create(ctx, gid, types.ItemCreate{ + Name: row.Name, + Description: row.Description, + LabelIDs: labelIDs, + LocationID: locationID, + }) + + if err != nil { + return err + } + + // Update the item with the rest of the data + _, err = svc.repo.Items.Update(ctx, types.ItemUpdate{ + ID: result.ID, + Name: result.Name, + LocationID: locationID, + LabelIDs: labelIDs, + Description: result.Description, + SerialNumber: row.SerialNumber, + ModelNumber: row.ModelNumber, + Manufacturer: row.Manufacturer, + Notes: row.Notes, + PurchaseFrom: row.PurchaseFrom, + PurchasePrice: row.parsedPurchasedPrice(), + PurchaseTime: row.parsedPurchasedAt(), + SoldTo: row.SoldTo, + SoldPrice: row.parsedSoldPrice(), + SoldTime: row.parsedSoldAt(), + }) + + if err != nil { + return err + } + } + return nil +} diff --git a/backend/internal/services/service_items_csv.go b/backend/internal/services/service_items_csv.go new file mode 100644 index 0000000..5e25127 --- /dev/null +++ b/backend/internal/services/service_items_csv.go @@ -0,0 +1,97 @@ +package services + +import ( + "errors" + "strconv" + "strings" + "time" +) + +var ErrInvalidCsv = errors.New("invalid csv") + +func parseFloat(s string) float64 { + if s == "" { + return 0 + } + f, _ := strconv.ParseFloat(s, 64) + return f +} + +func parseDate(s string) time.Time { + if s == "" { + return time.Time{} + } + + p, _ := time.Parse("01/02/2006", s) + return p +} + +type csvRow struct { + Location string + Labels string + Name string + Description string + SerialNumber string + ModelNumber string + Manufacturer string + Notes string + PurchaseFrom string + PurchasedPrice string + PurchasedAt string + SoldTo string + SoldPrice string + SoldAt string +} + +func newCsvRow(row []string) csvRow { + return csvRow{ + Location: row[0], + Labels: row[1], + Name: row[2], + Description: row[3], + SerialNumber: row[4], + ModelNumber: row[5], + Manufacturer: row[6], + Notes: row[7], + PurchaseFrom: row[8], + PurchasedPrice: row[9], + PurchasedAt: row[10], + SoldTo: row[11], + SoldPrice: row[12], + SoldAt: row[13], + } +} + +func (c csvRow) parsedSoldPrice() float64 { + return parseFloat(c.SoldPrice) +} + +func (c csvRow) parsedPurchasedPrice() float64 { + return parseFloat(c.PurchasedPrice) +} + +func (c csvRow) parsedPurchasedAt() time.Time { + return parseDate(c.PurchasedAt) +} + +func (c csvRow) parsedSoldAt() time.Time { + return parseDate(c.SoldAt) +} + +func (c csvRow) getLabels() []string { + split := strings.Split(c.Labels, ";") + + // Trim each + for i, s := range split { + split[i] = strings.TrimSpace(s) + } + + // Remove empty + for i, s := range split { + if s == "" { + split = append(split[:i], split[i+1:]...) + } + } + + return split +} diff --git a/backend/internal/services/service_items_csv_test.go b/backend/internal/services/service_items_csv_test.go new file mode 100644 index 0000000..34f35d0 --- /dev/null +++ b/backend/internal/services/service_items_csv_test.go @@ -0,0 +1,87 @@ +package services + +import ( + "bytes" + "encoding/csv" + "reflect" + "testing" +) + +const CSV_DATA = ` +Location,Labels,Name,Description,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased At,Sold To,Sold Price,Sold At +Garage,IOT;Home Assistant; Z-Wave,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,, +Living Room,IOT;Home Assistant; Z-Wave,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,, +Office,IOT;Home Assistant; Z-Wave,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,, +Downstairs,IOT;Home Assistant; Z-Wave,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,, +Entry,IOT;Home Assistant; Z-Wave,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,, +Kitchen,IOT;Home Assistant; Z-Wave,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,39351,Honeywell,,Amazon,65.98,09/30/0202,,, +` + +func loadcsv() [][]string { + reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA))) + + records, err := reader.ReadAll() + if err != nil { + panic(err) + } + + return records +} + +func Test_csvRow_getLabels(t *testing.T) { + type fields struct { + Labels string + } + tests := []struct { + name string + fields fields + want []string + }{ + { + name: "basic test", + fields: fields{ + Labels: "IOT;Home Assistant;Z-Wave", + }, + want: []string{"IOT", "Home Assistant", "Z-Wave"}, + }, + { + name: "no labels", + fields: fields{ + Labels: "", + }, + want: []string{}, + }, + { + name: "single label", + fields: fields{ + Labels: "IOT", + }, + want: []string{"IOT"}, + }, + { + name: "trailing semicolon", + fields: fields{ + Labels: "IOT;", + }, + want: []string{"IOT"}, + }, + + { + name: "whitespace", + fields: fields{ + Labels: " IOT; Home Assistant; Z-Wave ", + }, + want: []string{"IOT", "Home Assistant", "Z-Wave"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := csvRow{ + Labels: tt.fields.Labels, + } + if got := c.getLabels(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("csvRow.getLabels() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/internal/services/service_items_test.go b/backend/internal/services/service_items_test.go new file mode 100644 index 0000000..d5c18c0 --- /dev/null +++ b/backend/internal/services/service_items_test.go @@ -0,0 +1,84 @@ +package services + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + + + +func TestItemService_CsvImport(t *testing.T) { + data := loadcsv() + svc := &ItemService{ + repo: tRepos, + } + err := svc.CsvImport(context.Background(), tGroup.ID, data) + assert.NoError(t, err) + + items, err := svc.GetAll(context.Background(), tGroup.ID) + assert.NoError(t, err) + t.Cleanup(func() { + for _, item := range items { + err := svc.repo.Items.Delete(context.Background(), item.ID) + assert.NoError(t, err) + } + }) + + assert.Equal(t, len(items), 6) + + dataCsv := []csvRow{} + for _, item := range data { + dataCsv = append(dataCsv, newCsvRow(item)) + } + + locationService := &LocationService{ + repos: tRepos, + } + + LabelService := &LabelService{ + repos: tRepos, + } + + allLocation, err := locationService.GetAll(context.Background(), tGroup.ID) + assert.NoError(t, err) + locNames := []string{} + for _, loc := range allLocation { + locNames = append(locNames, loc.Name) + } + + allLabels, err := LabelService.GetAll(context.Background(), tGroup.ID) + assert.NoError(t, err) + labelNames := []string{} + for _, label := range allLabels { + labelNames = append(labelNames, label.Name) + } + + for _, item := range items { + assert.Contains(t, locNames, item.Location.Name) + for _, label := range item.Labels { + assert.Contains(t, labelNames, label.Name) + } + + for _, csvRow := range dataCsv { + if csvRow.Name == item.Name { + assert.Equal(t, csvRow.Description, item.Description) + assert.Equal(t, csvRow.SerialNumber, item.SerialNumber) + assert.Equal(t, csvRow.Manufacturer, item.Manufacturer) + assert.Equal(t, csvRow.Notes, item.Notes) + + // Purchase Fields + assert.Equal(t, csvRow.parsedPurchasedAt(), item.PurchaseTime) + assert.Equal(t, csvRow.PurchaseFrom, item.PurchaseFrom) + assert.Equal(t, csvRow.parsedPurchasedPrice(), item.PurchasePrice) + + // Sold Fields + assert.Equal(t, csvRow.parsedSoldAt(), item.SoldTime) + assert.Equal(t, csvRow.SoldTo, item.SoldTo) + assert.Equal(t, csvRow.parsedSoldPrice(), item.SoldPrice) + } + } + + } +}