diff --git a/backend/app/api/demo.go b/backend/app/api/demo.go
index adacfd8..a98f03d 100644
--- a/backend/app/api/demo.go
+++ b/backend/app/api/demo.go
@@ -52,7 +52,7 @@ func (a *app) SetupDemo() {
log.Fatal().Msg("Failed to setup demo")
}
- err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)
+ _, err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)
if err != nil {
log.Err(err).Msg("Failed to import CSV")
log.Fatal().Msg("Failed to setup demo")
diff --git a/backend/app/api/v1/v1_ctrl_items.go b/backend/app/api/v1/v1_ctrl_items.go
index 686842d..b515be9 100644
--- a/backend/app/api/v1/v1_ctrl_items.go
+++ b/backend/app/api/v1/v1_ctrl_items.go
@@ -220,7 +220,7 @@ func (ctrl *V1Controller) HandleItemsImport() http.HandlerFunc {
user := services.UseUserCtx(r.Context())
- err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)
+ _, err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)
if err != nil {
log.Err(err).Msg("failed to import items")
server.RespondServerError(w)
diff --git a/backend/internal/repo/repo_items.go b/backend/internal/repo/repo_items.go
index 64715ca..a0aa6f9 100644
--- a/backend/internal/repo/repo_items.go
+++ b/backend/internal/repo/repo_items.go
@@ -211,6 +211,11 @@ func (e *ItemsRepository) GetOne(ctx context.Context, id uuid.UUID) (ItemOut, er
return e.getOne(ctx, item.ID(id))
}
+func (e *ItemsRepository) CheckRef(ctx context.Context, GID uuid.UUID, ref string) (bool, error) {
+ q := e.db.Item.Query().Where(item.HasGroupWith(group.ID(GID)))
+ return q.Where(item.ImportRef(ref)).Exist(ctx)
+}
+
// GetOneByGroup returns a single item by ID. If the item does not exist, an error is returned.
// GetOneByGroup ensures that the item belongs to a specific group.
func (e *ItemsRepository) GetOneByGroup(ctx context.Context, gid, id uuid.UUID) (ItemOut, error) {
@@ -287,6 +292,7 @@ func (e *ItemsRepository) GetAll(ctx context.Context, gid uuid.UUID) ([]ItemSumm
func (e *ItemsRepository) Create(ctx context.Context, gid uuid.UUID, data ItemCreate) (ItemOut, error) {
q := e.db.Item.Create().
+ SetImportRef(data.ImportRef).
SetName(data.Name).
SetDescription(data.Description).
SetGroupID(gid).
diff --git a/backend/internal/services/service_items.go b/backend/internal/services/service_items.go
index e944e14..7650a97 100644
--- a/backend/internal/services/service_items.go
+++ b/backend/internal/services/service_items.go
@@ -3,7 +3,6 @@ package services
import (
"context"
"errors"
- "fmt"
"github.com/google/uuid"
"github.com/hay-kot/homebox/backend/internal/repo"
@@ -48,7 +47,7 @@ func (svc *ItemService) Update(ctx context.Context, gid uuid.UUID, data repo.Ite
return svc.repo.Items.UpdateByGroup(ctx, gid, data)
}
-func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) error {
+func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) (int, error) {
loaded := []csvRow{}
// Skip first row
@@ -59,18 +58,41 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
}
if len(row) != NumOfCols {
- return ErrInvalidCsv
+ return 0, ErrInvalidCsv
}
r := newCsvRow(row)
loaded = append(loaded, r)
}
+ // validate rows
+ var errMap = map[int][]error{}
+ var hasErr bool
+ for i, r := range loaded {
+
+ errs := r.validate()
+
+ if len(errs) > 0 {
+ hasErr = true
+ lineNum := i + 2
+
+ errMap[lineNum] = errs
+ }
+ }
+
+ if hasErr {
+ for lineNum, errs := range errMap {
+ for _, err := range errs {
+ log.Error().Err(err).Int("line", lineNum).Msg("csv import error")
+ }
+ }
+ }
+
// Bootstrap the locations and labels so we can reuse the created IDs for the items
locations := map[string]uuid.UUID{}
existingLocation, err := svc.repo.Locations.GetAll(ctx, gid)
if err != nil {
- return err
+ return 0, err
}
for _, loc := range existingLocation {
locations[loc.Name] = loc.ID
@@ -79,7 +101,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
labels := map[string]uuid.UUID{}
existingLabels, err := svc.repo.Labels.GetAll(ctx, gid)
if err != nil {
- return err
+ return 0, err
}
for _, label := range existingLabels {
labels[label.Name] = label.ID
@@ -88,25 +110,21 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
for _, row := range loaded {
// Locations
- if _, ok := locations[row.Location]; ok {
- continue
+ if _, exists := locations[row.Location]; !exists {
+ result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{
+ Name: row.Location,
+ Description: "",
+ })
+ if err != nil {
+ return 0, err
+ }
+ locations[row.Location] = result.ID
}
- fmt.Println("Creating Location: ", row.Location)
-
- result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{
- Name: row.Location,
- Description: "",
- })
- if err != nil {
- return err
- }
- locations[row.Location] = result.ID
-
// Labels
for _, label := range row.getLabels() {
- if _, ok := labels[label]; ok {
+ if _, exists := labels[label]; exists {
continue
}
result, err := svc.repo.Labels.Create(ctx, gid, repo.LabelCreate{
@@ -114,14 +132,26 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
Description: "",
})
if err != nil {
- return err
+ return 0, err
}
labels[label] = result.ID
}
}
// Create the items
+ var count int
for _, row := range loaded {
+ // Check Import Ref
+ if row.Item.ImportRef != "" {
+ exists, err := svc.repo.Items.CheckRef(ctx, gid, row.Item.ImportRef)
+ if exists {
+ continue
+ }
+ if err != nil {
+ log.Err(err).Msg("error checking import ref")
+ }
+ }
+
locationID := locations[row.Location]
labelIDs := []uuid.UUID{}
for _, label := range row.getLabels() {
@@ -131,8 +161,6 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
log.Info().
Str("name", row.Item.Name).
Str("location", row.Location).
- Strs("labels", row.getLabels()).
- Str("locationId", locationID.String()).
Msgf("Creating Item: %s", row.Item.Name)
result, err := svc.repo.Items.Create(ctx, gid, repo.ItemCreate{
@@ -144,7 +172,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
})
if err != nil {
- return err
+ return count, err
}
// Update the item with the rest of the data
@@ -183,8 +211,10 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
})
if err != nil {
- return err
+ return count, err
}
+
+ count++
}
- return nil
+ return count, nil
}
diff --git a/backend/internal/services/service_items_csv.go b/backend/internal/services/service_items_csv.go
index 4ad1aed..c559386 100644
--- a/backend/internal/services/service_items_csv.go
+++ b/backend/internal/services/service_items_csv.go
@@ -97,3 +97,22 @@ func (c csvRow) getLabels() []string {
return split
}
+
+func (c csvRow) validate() []error {
+ var errs []error
+
+ add := func(err error) {
+ errs = append(errs, err)
+ }
+
+ required := func(s string, name string) {
+ if s == "" {
+ add(errors.New(name + " is required"))
+ }
+ }
+
+ required(c.Location, "Location")
+ required(c.Item.Name, "Name")
+
+ return errs
+}
diff --git a/backend/internal/services/service_items_csv_test.go b/backend/internal/services/service_items_csv_test.go
index fed6e31..8c453e8 100644
--- a/backend/internal/services/service_items_csv_test.go
+++ b/backend/internal/services/service_items_csv_test.go
@@ -9,12 +9,12 @@ import (
const CSV_DATA = `
Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes
-,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
-,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,
-,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
-,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,
-,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,
-,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`
+A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
+B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,
+C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
+D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,
+E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,
+F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`
func loadcsv() [][]string {
reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA)))
diff --git a/backend/internal/services/service_items_test.go b/backend/internal/services/service_items_test.go
index 15fe3ca..018dbc1 100644
--- a/backend/internal/services/service_items_test.go
+++ b/backend/internal/services/service_items_test.go
@@ -13,7 +13,13 @@ func TestItemService_CsvImport(t *testing.T) {
svc := &ItemService{
repo: tRepos,
}
- err := svc.CsvImport(context.Background(), tGroup.ID, data)
+ count, err := svc.CsvImport(context.Background(), tGroup.ID, data)
+ assert.Equal(t, 6, count)
+ assert.NoError(t, err)
+
+ // Check import refs are deduplicated
+ count, err = svc.CsvImport(context.Background(), tGroup.ID, data)
+ assert.Equal(t, 0, count)
assert.NoError(t, err)
items, err := svc.GetAll(context.Background(), tGroup.ID)
diff --git a/docs/docs/import-csv.md b/docs/docs/import-csv.md
index 9c5e587..ea32b8f 100644
--- a/docs/docs/import-csv.md
+++ b/docs/docs/import-csv.md
@@ -23,29 +23,29 @@ Import RefLocation Labels Quantity Name Description Insured Serial Number Model
## CSV Reference
-| Column | Type | Description |
-| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------- |
-| ImportRef | String (100) | Future |
-| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. |
-| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new |
-| Quantity | Integer | The quantity of items to create |
-| Name | String | Name of the item |
-| Description | String | Description of the item |
-| Insured | Boolean | Whether or not the item is insured |
-| Serial Number | String | Serial number of the item |
-| Model Number | String | Model of the item |
-| Manufacturer | String | Manufacturer of the item |
-| Notes | String (1000) | General notes about the product |
-| Purchase From | String | Name of the place the item was purchased from |
-| Purchase Price | Float64 | |
-| Purchase At | Date | Date the item was purchased |
-| Lifetime Warranty | Boolean | true or false - case insensitive |
-| Warranty Expires | Date | Date in the format |
-| Warranty Details | String | Details about the warranty |
-| Sold To | String | Name of the person the item was sold to |
-| Sold At | Date | Date the item was sold |
-| Sold Price | Float64 | |
-| Sold Notes | String (1000) | |
+| Column | Type | Description |
+| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| ImportRef | String (100) | Import Refs are unique strings that can be used to deduplicate imports. Before an item is imported, we check the database for a matching ref. If the ref exists, we skip that item. |
+| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. |
+| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new |
+| Quantity | Integer | The quantity of items to create |
+| Name | String | Name of the item |
+| Description | String | Description of the item |
+| Insured | Boolean | Whether or not the item is insured |
+| Serial Number | String | Serial number of the item |
+| Model Number | String | Model of the item |
+| Manufacturer | String | Manufacturer of the item |
+| Notes | String (1000) | General notes about the product |
+| Purchase From | String | Name of the place the item was purchased from |
+| Purchase Price | Float64 | |
+| Purchase At | Date | Date the item was purchased |
+| Lifetime Warranty | Boolean | true or false - case insensitive |
+| Warranty Expires | Date | Date in the format |
+| Warranty Details | String | Details about the warranty |
+| Sold To | String | Name of the person the item was sold to |
+| Sold At | Date | Date the item was sold |
+| Sold Price | Float64 | |
+| Sold Notes | String (1000) | |
**Type Key**