fix import bug and add ref support (#88)

* fix import bug and add ref support

* fix calls

* add docs

backend/app/api/demo.go

@@ -52,7 +52,7 @@ func (a *app) SetupDemo() {
 		log.Fatal().Msg("Failed to setup demo")
 	}
 
-	err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)
+	_, err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)
 	if err != nil {
 		log.Err(err).Msg("Failed to import CSV")
 		log.Fatal().Msg("Failed to setup demo")

backend/app/api/v1/v1_ctrl_items.go

@@ -220,7 +220,7 @@ func (ctrl *V1Controller) HandleItemsImport() http.HandlerFunc {
 
 		user := services.UseUserCtx(r.Context())
 
-		err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)
+		_, err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)
 		if err != nil {
 			log.Err(err).Msg("failed to import items")
 			server.RespondServerError(w)

backend/internal/repo/repo_items.go

@@ -211,6 +211,11 @@ func (e *ItemsRepository) GetOne(ctx context.Context, id uuid.UUID) (ItemOut, er
 	return e.getOne(ctx, item.ID(id))
 }
 
+func (e *ItemsRepository) CheckRef(ctx context.Context, GID uuid.UUID, ref string) (bool, error) {
+	q := e.db.Item.Query().Where(item.HasGroupWith(group.ID(GID)))
+	return q.Where(item.ImportRef(ref)).Exist(ctx)
+}
+
 // GetOneByGroup returns a single item by ID. If the item does not exist, an error is returned.
 // GetOneByGroup ensures that the item belongs to a specific group.
 func (e *ItemsRepository) GetOneByGroup(ctx context.Context, gid, id uuid.UUID) (ItemOut, error) {
@@ -287,6 +292,7 @@ func (e *ItemsRepository) GetAll(ctx context.Context, gid uuid.UUID) ([]ItemSumm
 
 func (e *ItemsRepository) Create(ctx context.Context, gid uuid.UUID, data ItemCreate) (ItemOut, error) {
 	q := e.db.Item.Create().
+		SetImportRef(data.ImportRef).
 		SetName(data.Name).
 		SetDescription(data.Description).
 		SetGroupID(gid).

backend/internal/services/service_items.go

@@ -3,7 +3,6 @@ package services
 import (
 	"context"
 	"errors"
-	"fmt"
 
 	"github.com/google/uuid"
 	"github.com/hay-kot/homebox/backend/internal/repo"
@@ -48,7 +47,7 @@ func (svc *ItemService) Update(ctx context.Context, gid uuid.UUID, data repo.Ite
 	return svc.repo.Items.UpdateByGroup(ctx, gid, data)
 }
 
-func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) error {
+func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) (int, error) {
 	loaded := []csvRow{}
 
 	// Skip first row
@@ -59,18 +58,41 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
 		}
 
 		if len(row) != NumOfCols {
-			return ErrInvalidCsv
+			return 0, ErrInvalidCsv
 		}
 
 		r := newCsvRow(row)
 		loaded = append(loaded, r)
 	}
 
+	// validate rows
+	var errMap = map[int][]error{}
+	var hasErr bool
+	for i, r := range loaded {
+
+		errs := r.validate()
+
+		if len(errs) > 0 {
+			hasErr = true
+			lineNum := i + 2
+
+			errMap[lineNum] = errs
+		}
+	}
+
+	if hasErr {
+		for lineNum, errs := range errMap {
+			for _, err := range errs {
+				log.Error().Err(err).Int("line", lineNum).Msg("csv import error")
+			}
+		}
+	}
+
 	// Bootstrap the locations and labels so we can reuse the created IDs for the items
 	locations := map[string]uuid.UUID{}
 	existingLocation, err := svc.repo.Locations.GetAll(ctx, gid)
 	if err != nil {
-		return err
+		return 0, err
 	}
 	for _, loc := range existingLocation {
 		locations[loc.Name] = loc.ID
@@ -79,7 +101,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
 	labels := map[string]uuid.UUID{}
 	existingLabels, err := svc.repo.Labels.GetAll(ctx, gid)
 	if err != nil {
-		return err
+		return 0, err
 	}
 	for _, label := range existingLabels {
 		labels[label.Name] = label.ID
@@ -88,25 +110,21 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
 	for _, row := range loaded {
 
 		// Locations
-		if _, ok := locations[row.Location]; ok {
+		if _, exists := locations[row.Location]; !exists {
-			continue
-		}
-
-		fmt.Println("Creating Location: ", row.Location)
-
 			result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{
 				Name:        row.Location,
 				Description: "",
 			})
 			if err != nil {
-			return err
+				return 0, err
 			}
 			locations[row.Location] = result.ID
+		}
 
 		// Labels
 
 		for _, label := range row.getLabels() {
-			if _, ok := labels[label]; ok {
+			if _, exists := labels[label]; exists {
 				continue
 			}
 			result, err := svc.repo.Labels.Create(ctx, gid, repo.LabelCreate{
@@ -114,14 +132,26 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
 				Description: "",
 			})
 			if err != nil {
-				return err
+				return 0, err
 			}
 			labels[label] = result.ID
 		}
 	}
 
 	// Create the items
+	var count int
 	for _, row := range loaded {
+		// Check Import Ref
+		if row.Item.ImportRef != "" {
+			exists, err := svc.repo.Items.CheckRef(ctx, gid, row.Item.ImportRef)
+			if exists {
+				continue
+			}
+			if err != nil {
+				log.Err(err).Msg("error checking import ref")
+			}
+		}
+
 		locationID := locations[row.Location]
 		labelIDs := []uuid.UUID{}
 		for _, label := range row.getLabels() {
@@ -131,8 +161,6 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
 		log.Info().
 			Str("name", row.Item.Name).
 			Str("location", row.Location).
-			Strs("labels", row.getLabels()).
-			Str("locationId", locationID.String()).
 			Msgf("Creating Item: %s", row.Item.Name)
 
 		result, err := svc.repo.Items.Create(ctx, gid, repo.ItemCreate{
@@ -144,7 +172,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
 		})
 
 		if err != nil {
-			return err
+			return count, err
 		}
 
 		// Update the item with the rest of the data
@@ -183,8 +211,10 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
 		})
 
 		if err != nil {
-			return err
+			return count, err
 		}
+
+		count++
 	}
-	return nil
+	return count, nil
 }

backend/internal/services/service_items_csv.go

@@ -97,3 +97,22 @@ func (c csvRow) getLabels() []string {
 
 	return split
 }
+
+func (c csvRow) validate() []error {
+	var errs []error
+
+	add := func(err error) {
+		errs = append(errs, err)
+	}
+
+	required := func(s string, name string) {
+		if s == "" {
+			add(errors.New(name + " is required"))
+		}
+	}
+
+	required(c.Location, "Location")
+	required(c.Item.Name, "Name")
+
+	return errs
+}

backend/internal/services/service_items_csv_test.go

@@ -9,12 +9,12 @@ import (
 
 const CSV_DATA = `
 Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes
-,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
+A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
-,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,
+B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,
-,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
+C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
-,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,
+D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,
-,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,
+E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,
-,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`
+F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`
 
 func loadcsv() [][]string {
 	reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA)))

backend/internal/services/service_items_test.go

@@ -13,7 +13,13 @@ func TestItemService_CsvImport(t *testing.T) {
 	svc := &ItemService{
 		repo: tRepos,
 	}
-	err := svc.CsvImport(context.Background(), tGroup.ID, data)
+	count, err := svc.CsvImport(context.Background(), tGroup.ID, data)
+	assert.Equal(t, 6, count)
+	assert.NoError(t, err)
+
+	// Check import refs are deduplicated
+	count, err = svc.CsvImport(context.Background(), tGroup.ID, data)
+	assert.Equal(t, 0, count)
 	assert.NoError(t, err)
 
 	items, err := svc.GetAll(context.Background(), tGroup.ID)

docs/docs/import-csv.md

@@ -24,8 +24,8 @@ Import RefLocation	Labels	Quantity	Name	Description	Insured	Serial Number	Model
 ## CSV Reference
 
 | Column            | Type                 | Description                                                                                                                                                                         |
-| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------- |
+| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| ImportRef         | String (100)         | Future                                                                                                                        |
+| ImportRef         | String (100)         | Import Refs are unique strings that can be used to deduplicate imports. Before an item is imported, we check the database for a matching ref. If the ref exists, we skip that item. |
 | Location          | String               | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused.                                                       |
 | Labels            | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new                                                                                                      |
 | Quantity          | Integer              | The quantity of items to create                                                                                                                                                     |