From dbaaf4ad0a5e57751cca4f86f0da45c196a54a65 Mon Sep 17 00:00:00 2001 From: Hayden <64056131+hay-kot@users.noreply.github.com> Date: Sat, 15 Oct 2022 17:46:57 -0800 Subject: [PATCH] fix import bug and add ref support (#88) * fix import bug and add ref support * fix calls * add docs --- backend/app/api/demo.go | 2 +- backend/app/api/v1/v1_ctrl_items.go | 2 +- backend/internal/repo/repo_items.go | 6 ++ backend/internal/services/service_items.go | 80 +++++++++++++------ .../internal/services/service_items_csv.go | 19 +++++ .../services/service_items_csv_test.go | 12 +-- .../internal/services/service_items_test.go | 8 +- docs/docs/import-csv.md | 46 +++++------ 8 files changed, 118 insertions(+), 57 deletions(-) diff --git a/backend/app/api/demo.go b/backend/app/api/demo.go index adacfd8..a98f03d 100644 --- a/backend/app/api/demo.go +++ b/backend/app/api/demo.go @@ -52,7 +52,7 @@ func (a *app) SetupDemo() { log.Fatal().Msg("Failed to setup demo") } - err = a.services.Items.CsvImport(context.Background(), self.GroupID, records) + _, err = a.services.Items.CsvImport(context.Background(), self.GroupID, records) if err != nil { log.Err(err).Msg("Failed to import CSV") log.Fatal().Msg("Failed to setup demo") diff --git a/backend/app/api/v1/v1_ctrl_items.go b/backend/app/api/v1/v1_ctrl_items.go index 686842d..b515be9 100644 --- a/backend/app/api/v1/v1_ctrl_items.go +++ b/backend/app/api/v1/v1_ctrl_items.go @@ -220,7 +220,7 @@ func (ctrl *V1Controller) HandleItemsImport() http.HandlerFunc { user := services.UseUserCtx(r.Context()) - err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data) + _, err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data) if err != nil { log.Err(err).Msg("failed to import items") server.RespondServerError(w) diff --git a/backend/internal/repo/repo_items.go b/backend/internal/repo/repo_items.go index 64715ca..a0aa6f9 100644 --- a/backend/internal/repo/repo_items.go +++ b/backend/internal/repo/repo_items.go @@ -211,6 +211,11 @@ func (e *ItemsRepository) GetOne(ctx context.Context, id uuid.UUID) (ItemOut, er return e.getOne(ctx, item.ID(id)) } +func (e *ItemsRepository) CheckRef(ctx context.Context, GID uuid.UUID, ref string) (bool, error) { + q := e.db.Item.Query().Where(item.HasGroupWith(group.ID(GID))) + return q.Where(item.ImportRef(ref)).Exist(ctx) +} + // GetOneByGroup returns a single item by ID. If the item does not exist, an error is returned. // GetOneByGroup ensures that the item belongs to a specific group. func (e *ItemsRepository) GetOneByGroup(ctx context.Context, gid, id uuid.UUID) (ItemOut, error) { @@ -287,6 +292,7 @@ func (e *ItemsRepository) GetAll(ctx context.Context, gid uuid.UUID) ([]ItemSumm func (e *ItemsRepository) Create(ctx context.Context, gid uuid.UUID, data ItemCreate) (ItemOut, error) { q := e.db.Item.Create(). + SetImportRef(data.ImportRef). SetName(data.Name). SetDescription(data.Description). SetGroupID(gid). diff --git a/backend/internal/services/service_items.go b/backend/internal/services/service_items.go index e944e14..7650a97 100644 --- a/backend/internal/services/service_items.go +++ b/backend/internal/services/service_items.go @@ -3,7 +3,6 @@ package services import ( "context" "errors" - "fmt" "github.com/google/uuid" "github.com/hay-kot/homebox/backend/internal/repo" @@ -48,7 +47,7 @@ func (svc *ItemService) Update(ctx context.Context, gid uuid.UUID, data repo.Ite return svc.repo.Items.UpdateByGroup(ctx, gid, data) } -func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) error { +func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) (int, error) { loaded := []csvRow{} // Skip first row @@ -59,18 +58,41 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s } if len(row) != NumOfCols { - return ErrInvalidCsv + return 0, ErrInvalidCsv } r := newCsvRow(row) loaded = append(loaded, r) } + // validate rows + var errMap = map[int][]error{} + var hasErr bool + for i, r := range loaded { + + errs := r.validate() + + if len(errs) > 0 { + hasErr = true + lineNum := i + 2 + + errMap[lineNum] = errs + } + } + + if hasErr { + for lineNum, errs := range errMap { + for _, err := range errs { + log.Error().Err(err).Int("line", lineNum).Msg("csv import error") + } + } + } + // Bootstrap the locations and labels so we can reuse the created IDs for the items locations := map[string]uuid.UUID{} existingLocation, err := svc.repo.Locations.GetAll(ctx, gid) if err != nil { - return err + return 0, err } for _, loc := range existingLocation { locations[loc.Name] = loc.ID @@ -79,7 +101,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s labels := map[string]uuid.UUID{} existingLabels, err := svc.repo.Labels.GetAll(ctx, gid) if err != nil { - return err + return 0, err } for _, label := range existingLabels { labels[label.Name] = label.ID @@ -88,25 +110,21 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s for _, row := range loaded { // Locations - if _, ok := locations[row.Location]; ok { - continue + if _, exists := locations[row.Location]; !exists { + result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{ + Name: row.Location, + Description: "", + }) + if err != nil { + return 0, err + } + locations[row.Location] = result.ID } - fmt.Println("Creating Location: ", row.Location) - - result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{ - Name: row.Location, - Description: "", - }) - if err != nil { - return err - } - locations[row.Location] = result.ID - // Labels for _, label := range row.getLabels() { - if _, ok := labels[label]; ok { + if _, exists := labels[label]; exists { continue } result, err := svc.repo.Labels.Create(ctx, gid, repo.LabelCreate{ @@ -114,14 +132,26 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s Description: "", }) if err != nil { - return err + return 0, err } labels[label] = result.ID } } // Create the items + var count int for _, row := range loaded { + // Check Import Ref + if row.Item.ImportRef != "" { + exists, err := svc.repo.Items.CheckRef(ctx, gid, row.Item.ImportRef) + if exists { + continue + } + if err != nil { + log.Err(err).Msg("error checking import ref") + } + } + locationID := locations[row.Location] labelIDs := []uuid.UUID{} for _, label := range row.getLabels() { @@ -131,8 +161,6 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s log.Info(). Str("name", row.Item.Name). Str("location", row.Location). - Strs("labels", row.getLabels()). - Str("locationId", locationID.String()). Msgf("Creating Item: %s", row.Item.Name) result, err := svc.repo.Items.Create(ctx, gid, repo.ItemCreate{ @@ -144,7 +172,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s }) if err != nil { - return err + return count, err } // Update the item with the rest of the data @@ -183,8 +211,10 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s }) if err != nil { - return err + return count, err } + + count++ } - return nil + return count, nil } diff --git a/backend/internal/services/service_items_csv.go b/backend/internal/services/service_items_csv.go index 4ad1aed..c559386 100644 --- a/backend/internal/services/service_items_csv.go +++ b/backend/internal/services/service_items_csv.go @@ -97,3 +97,22 @@ func (c csvRow) getLabels() []string { return split } + +func (c csvRow) validate() []error { + var errs []error + + add := func(err error) { + errs = append(errs, err) + } + + required := func(s string, name string) { + if s == "" { + add(errors.New(name + " is required")) + } + } + + required(c.Location, "Location") + required(c.Item.Name, "Name") + + return errs +} diff --git a/backend/internal/services/service_items_csv_test.go b/backend/internal/services/service_items_csv_test.go index fed6e31..8c453e8 100644 --- a/backend/internal/services/service_items_csv_test.go +++ b/backend/internal/services/service_items_csv_test.go @@ -9,12 +9,12 @@ import ( const CSV_DATA = ` Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes -,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,, -,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,, -,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,, -,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,, -,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,, -,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,` +A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,, +B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,, +C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,, +D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,, +E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,, +F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,` func loadcsv() [][]string { reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA))) diff --git a/backend/internal/services/service_items_test.go b/backend/internal/services/service_items_test.go index 15fe3ca..018dbc1 100644 --- a/backend/internal/services/service_items_test.go +++ b/backend/internal/services/service_items_test.go @@ -13,7 +13,13 @@ func TestItemService_CsvImport(t *testing.T) { svc := &ItemService{ repo: tRepos, } - err := svc.CsvImport(context.Background(), tGroup.ID, data) + count, err := svc.CsvImport(context.Background(), tGroup.ID, data) + assert.Equal(t, 6, count) + assert.NoError(t, err) + + // Check import refs are deduplicated + count, err = svc.CsvImport(context.Background(), tGroup.ID, data) + assert.Equal(t, 0, count) assert.NoError(t, err) items, err := svc.GetAll(context.Background(), tGroup.ID) diff --git a/docs/docs/import-csv.md b/docs/docs/import-csv.md index 9c5e587..ea32b8f 100644 --- a/docs/docs/import-csv.md +++ b/docs/docs/import-csv.md @@ -23,29 +23,29 @@ Import RefLocation Labels Quantity Name Description Insured Serial Number Model ## CSV Reference -| Column | Type | Description | -| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| ImportRef | String (100) | Future | -| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. | -| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new | -| Quantity | Integer | The quantity of items to create | -| Name | String | Name of the item | -| Description | String | Description of the item | -| Insured | Boolean | Whether or not the item is insured | -| Serial Number | String | Serial number of the item | -| Model Number | String | Model of the item | -| Manufacturer | String | Manufacturer of the item | -| Notes | String (1000) | General notes about the product | -| Purchase From | String | Name of the place the item was purchased from | -| Purchase Price | Float64 | | -| Purchase At | Date | Date the item was purchased | -| Lifetime Warranty | Boolean | true or false - case insensitive | -| Warranty Expires | Date | Date in the format | -| Warranty Details | String | Details about the warranty | -| Sold To | String | Name of the person the item was sold to | -| Sold At | Date | Date the item was sold | -| Sold Price | Float64 | | -| Sold Notes | String (1000) | | +| Column | Type | Description | +| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ImportRef | String (100) | Import Refs are unique strings that can be used to deduplicate imports. Before an item is imported, we check the database for a matching ref. If the ref exists, we skip that item. | +| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. | +| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new | +| Quantity | Integer | The quantity of items to create | +| Name | String | Name of the item | +| Description | String | Description of the item | +| Insured | Boolean | Whether or not the item is insured | +| Serial Number | String | Serial number of the item | +| Model Number | String | Model of the item | +| Manufacturer | String | Manufacturer of the item | +| Notes | String (1000) | General notes about the product | +| Purchase From | String | Name of the place the item was purchased from | +| Purchase Price | Float64 | | +| Purchase At | Date | Date the item was purchased | +| Lifetime Warranty | Boolean | true or false - case insensitive | +| Warranty Expires | Date | Date in the format | +| Warranty Details | String | Details about the warranty | +| Sold To | String | Name of the person the item was sold to | +| Sold At | Date | Date the item was sold | +| Sold Price | Float64 | | +| Sold Notes | String (1000) | | **Type Key**