forked from mirrors/homebox
feat: import export rewrite (#290)
* WIP: initial work * refactoring * fix failing JS tests * update import docs * fix import headers * fix column headers * update refs on import * remove demo status * finnnneeeee * formatting
This commit is contained in:
Hayden
GitHub
parent
a005fa5b9b
commit
a6bcb36c5b
41 changed files with 1616 additions and 796 deletions
|
|
@ -3,10 +3,13 @@ package services
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/hay-kot/homebox/backend/internal/core/services/reporting"
|
||||
"github.com/hay-kot/homebox/backend/internal/data/repo"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -37,7 +40,6 @@ func (svc *ItemService) Create(ctx Context, item repo.ItemCreate) (repo.ItemOut,
|
|||
|
||||
func (svc *ItemService) EnsureAssetID(ctx context.Context, GID uuid.UUID) (int, error) {
|
||||
items, err := svc.repo.Items.GetAllZeroAssetID(ctx, GID)
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
@ -61,190 +63,290 @@ func (svc *ItemService) EnsureAssetID(ctx context.Context, GID uuid.UUID) (int,
|
|||
|
||||
return finished, nil
|
||||
}
|
||||
func (svc *ItemService) CsvImport(ctx context.Context, GID uuid.UUID, data [][]string) (int, error) {
|
||||
loaded := []csvRow{}
|
||||
|
||||
// Skip first row
|
||||
for _, row := range data[1:] {
|
||||
// Skip empty rows
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(row) != NumOfCols {
|
||||
return 0, ErrInvalidCsv
|
||||
}
|
||||
|
||||
r := newCsvRow(row)
|
||||
loaded = append(loaded, r)
|
||||
}
|
||||
|
||||
// validate rows
|
||||
var errMap = map[int][]error{}
|
||||
var hasErr bool
|
||||
for i, r := range loaded {
|
||||
|
||||
errs := r.validate()
|
||||
|
||||
if len(errs) > 0 {
|
||||
hasErr = true
|
||||
lineNum := i + 2
|
||||
|
||||
errMap[lineNum] = errs
|
||||
}
|
||||
}
|
||||
|
||||
if hasErr {
|
||||
for lineNum, errs := range errMap {
|
||||
for _, err := range errs {
|
||||
log.Error().Err(err).Int("line", lineNum).Msg("csv import error")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bootstrap the locations and labels so we can reuse the created IDs for the items
|
||||
locations := map[string]uuid.UUID{}
|
||||
existingLocation, err := svc.repo.Locations.GetAll(ctx, GID, repo.LocationQuery{})
|
||||
func (svc *ItemService) EnsureImportRef(ctx context.Context, GID uuid.UUID) (int, error) {
|
||||
ids, err := svc.repo.Items.GetAllZeroImportRef(ctx, GID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, loc := range existingLocation {
|
||||
locations[loc.Name] = loc.ID
|
||||
|
||||
finished := 0
|
||||
for _, itemID := range ids {
|
||||
ref := uuid.New().String()[0:8]
|
||||
|
||||
err = svc.repo.Items.Patch(ctx, GID, itemID, repo.ItemPatch{ImportRef: &ref})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
finished++
|
||||
}
|
||||
|
||||
labels := map[string]uuid.UUID{}
|
||||
existingLabels, err := svc.repo.Labels.GetAll(ctx, GID)
|
||||
return finished, nil
|
||||
}
|
||||
|
||||
func serializeLocation[T ~[]string](location T) string {
|
||||
return strings.Join(location, "/")
|
||||
}
|
||||
|
||||
// CsvImport imports items from a CSV file. using the standard defined format.
|
||||
//
|
||||
// CsvImport applies the following rules/operations
|
||||
//
|
||||
// 1. If the item does not exist, it is created.
|
||||
// 2. If the item has a ImportRef and it exists it is skipped
|
||||
// 3. Locations and Labels are created if they do not exist.
|
||||
func (svc *ItemService) CsvImport(ctx context.Context, GID uuid.UUID, data io.Reader) (int, error) {
|
||||
sheet := reporting.IOSheet{}
|
||||
|
||||
err := sheet.Read(data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, label := range existingLabels {
|
||||
labels[label.Name] = label.ID
|
||||
}
|
||||
|
||||
for _, row := range loaded {
|
||||
// ========================================
|
||||
// Labels
|
||||
|
||||
// Locations
|
||||
if _, exists := locations[row.Location]; !exists {
|
||||
result, err := svc.repo.Locations.Create(ctx, GID, repo.LocationCreate{
|
||||
Name: row.Location,
|
||||
Description: "",
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
locations[row.Location] = result.ID
|
||||
labelMap := make(map[string]uuid.UUID)
|
||||
{
|
||||
labels, err := svc.repo.Labels.GetAll(ctx, GID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Labels
|
||||
|
||||
for _, label := range row.getLabels() {
|
||||
if _, exists := labels[label]; exists {
|
||||
continue
|
||||
}
|
||||
result, err := svc.repo.Labels.Create(ctx, GID, repo.LabelCreate{
|
||||
Name: label,
|
||||
Description: "",
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
labels[label] = result.ID
|
||||
for _, label := range labels {
|
||||
labelMap[label.Name] = label.ID
|
||||
}
|
||||
}
|
||||
|
||||
highest := repo.AssetID(-1)
|
||||
// ========================================
|
||||
// Locations
|
||||
|
||||
locationMap := make(map[string]uuid.UUID)
|
||||
{
|
||||
locations, err := svc.repo.Locations.Tree(ctx, GID, repo.TreeQuery{WithItems: false})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Traverse the tree and build a map of location full paths to IDs
|
||||
// where the full path is the location name joined by slashes.
|
||||
var traverse func(location *repo.TreeItem, path []string)
|
||||
traverse = func(location *repo.TreeItem, path []string) {
|
||||
path = append(path, location.Name)
|
||||
|
||||
locationMap[serializeLocation(path)] = location.ID
|
||||
|
||||
for _, child := range location.Children {
|
||||
traverse(child, path)
|
||||
}
|
||||
}
|
||||
|
||||
for _, location := range locations {
|
||||
traverse(&location, []string{})
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================
|
||||
// Import items
|
||||
|
||||
// Asset ID Pre-Check
|
||||
highestAID := repo.AssetID(-1)
|
||||
if svc.autoIncrementAssetID {
|
||||
highest, err = svc.repo.Items.GetHighestAssetID(ctx, GID)
|
||||
highestAID, err = svc.repo.Items.GetHighestAssetID(ctx, GID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// Create the items
|
||||
var count int
|
||||
for _, row := range loaded {
|
||||
// Check Import Ref
|
||||
if row.Item.ImportRef != "" {
|
||||
exists, err := svc.repo.Items.CheckRef(ctx, GID, row.Item.ImportRef)
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
finished := 0
|
||||
|
||||
for i := range sheet.Rows {
|
||||
row := sheet.Rows[i]
|
||||
|
||||
createRequired := true
|
||||
|
||||
// ========================================
|
||||
// Preflight check for existing item
|
||||
if row.ImportRef != "" {
|
||||
exists, err := svc.repo.Items.CheckRef(ctx, GID, row.ImportRef)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("error checking import ref")
|
||||
return 0, fmt.Errorf("error checking for existing item with ref %q: %w", row.ImportRef, err)
|
||||
}
|
||||
|
||||
if exists {
|
||||
createRequired = false
|
||||
}
|
||||
}
|
||||
|
||||
locationID := locations[row.Location]
|
||||
labelIDs := []uuid.UUID{}
|
||||
for _, label := range row.getLabels() {
|
||||
labelIDs = append(labelIDs, labels[label])
|
||||
// ========================================
|
||||
// Pre-Create Labels as necessary
|
||||
labelIds := make([]uuid.UUID, len(row.LabelStr))
|
||||
|
||||
for j := range row.LabelStr {
|
||||
label := row.LabelStr[j]
|
||||
|
||||
id, ok := labelMap[label]
|
||||
if !ok {
|
||||
newLabel, err := svc.repo.Labels.Create(ctx, GID, repo.LabelCreate{Name: label})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
id = newLabel.ID
|
||||
}
|
||||
|
||||
labelIds[j] = id
|
||||
labelMap[label] = id
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Str("name", row.Item.Name).
|
||||
Str("location", row.Location).
|
||||
Msgf("Creating Item: %s", row.Item.Name)
|
||||
// ========================================
|
||||
// Pre-Create Locations as necessary
|
||||
path := serializeLocation(row.Location)
|
||||
|
||||
data := repo.ItemCreate{
|
||||
ImportRef: row.Item.ImportRef,
|
||||
Name: row.Item.Name,
|
||||
Description: row.Item.Description,
|
||||
LabelIDs: labelIDs,
|
||||
LocationID: locationID,
|
||||
locationID, ok := locationMap[path]
|
||||
if !ok { // Traverse the path of LocationStr and check each path element to see if it exists already, if not create it.
|
||||
paths := []string{}
|
||||
for i, pathElement := range row.Location {
|
||||
paths = append(paths, pathElement)
|
||||
path := serializeLocation(paths)
|
||||
|
||||
locationID, ok = locationMap[path]
|
||||
if !ok {
|
||||
parentID := uuid.Nil
|
||||
|
||||
// Get the parent ID
|
||||
if i > 0 {
|
||||
parentPath := serializeLocation(row.Location[:i])
|
||||
parentID = locationMap[parentPath]
|
||||
}
|
||||
|
||||
newLocation, err := svc.repo.Locations.Create(ctx, GID, repo.LocationCreate{
|
||||
ParentID: parentID,
|
||||
Name: pathElement,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
locationID = newLocation.ID
|
||||
}
|
||||
|
||||
locationMap[path] = locationID
|
||||
}
|
||||
|
||||
locationID, ok = locationMap[path]
|
||||
if !ok {
|
||||
return 0, errors.New("failed to create location")
|
||||
}
|
||||
}
|
||||
|
||||
if svc.autoIncrementAssetID {
|
||||
highest++
|
||||
data.AssetID = highest
|
||||
var effAID repo.AssetID
|
||||
if svc.autoIncrementAssetID && row.AssetID.Nil() {
|
||||
effAID = highestAID + 1
|
||||
highestAID++
|
||||
} else {
|
||||
effAID = row.AssetID
|
||||
}
|
||||
|
||||
result, err := svc.repo.Items.Create(ctx, GID, data)
|
||||
// ========================================
|
||||
// Create Item
|
||||
var item repo.ItemOut
|
||||
switch {
|
||||
case createRequired:
|
||||
newItem := repo.ItemCreate{
|
||||
ImportRef: row.ImportRef,
|
||||
Name: row.Name,
|
||||
Description: row.Description,
|
||||
AssetID: effAID,
|
||||
LocationID: locationID,
|
||||
LabelIDs: labelIds,
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return count, err
|
||||
item, err = svc.repo.Items.Create(ctx, GID, newItem)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
default:
|
||||
item, err = svc.repo.Items.GetByRef(ctx, GID, row.ImportRef)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// Update the item with the rest of the data
|
||||
_, err = svc.repo.Items.UpdateByGroup(ctx, GID, repo.ItemUpdate{
|
||||
// Edges
|
||||
if item.ID == uuid.Nil {
|
||||
panic("item ID is nil on import - this should never happen")
|
||||
}
|
||||
|
||||
fields := make([]repo.ItemField, len(row.Fields))
|
||||
for i := range row.Fields {
|
||||
fields[i] = repo.ItemField{
|
||||
Name: row.Fields[i].Name,
|
||||
Type: "text",
|
||||
TextValue: row.Fields[i].Value,
|
||||
}
|
||||
}
|
||||
|
||||
updateItem := repo.ItemUpdate{
|
||||
ID: item.ID,
|
||||
LabelIDs: labelIds,
|
||||
LocationID: locationID,
|
||||
LabelIDs: labelIDs,
|
||||
AssetID: data.AssetID,
|
||||
|
||||
// General Fields
|
||||
ID: result.ID,
|
||||
Name: result.Name,
|
||||
Description: result.Description,
|
||||
Insured: row.Item.Insured,
|
||||
Notes: row.Item.Notes,
|
||||
Quantity: row.Item.Quantity,
|
||||
Name: row.Name,
|
||||
Description: row.Description,
|
||||
AssetID: effAID,
|
||||
Insured: row.Insured,
|
||||
Quantity: row.Quantity,
|
||||
Archived: row.Archived,
|
||||
|
||||
// Identifies the item as imported
|
||||
SerialNumber: row.Item.SerialNumber,
|
||||
ModelNumber: row.Item.ModelNumber,
|
||||
Manufacturer: row.Item.Manufacturer,
|
||||
PurchasePrice: row.PurchasePrice,
|
||||
PurchaseFrom: row.PurchaseFrom,
|
||||
PurchaseTime: row.PurchaseTime,
|
||||
|
||||
// Purchase
|
||||
PurchaseFrom: row.Item.PurchaseFrom,
|
||||
PurchasePrice: row.Item.PurchasePrice,
|
||||
PurchaseTime: row.Item.PurchaseTime,
|
||||
Manufacturer: row.Manufacturer,
|
||||
ModelNumber: row.ModelNumber,
|
||||
SerialNumber: row.SerialNumber,
|
||||
|
||||
// Warranty
|
||||
LifetimeWarranty: row.Item.LifetimeWarranty,
|
||||
WarrantyExpires: row.Item.WarrantyExpires,
|
||||
WarrantyDetails: row.Item.WarrantyDetails,
|
||||
LifetimeWarranty: row.LifetimeWarranty,
|
||||
WarrantyExpires: row.WarrantyExpires,
|
||||
WarrantyDetails: row.WarrantyDetails,
|
||||
|
||||
SoldTo: row.Item.SoldTo,
|
||||
SoldPrice: row.Item.SoldPrice,
|
||||
SoldTime: row.Item.SoldTime,
|
||||
SoldNotes: row.Item.SoldNotes,
|
||||
})
|
||||
SoldTo: row.SoldTo,
|
||||
SoldTime: row.SoldTime,
|
||||
SoldPrice: row.SoldPrice,
|
||||
SoldNotes: row.SoldNotes,
|
||||
|
||||
if err != nil {
|
||||
return count, err
|
||||
Notes: row.Notes,
|
||||
Fields: fields,
|
||||
}
|
||||
|
||||
count++
|
||||
item, err = svc.repo.Items.UpdateByGroup(ctx, GID, updateItem)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
finished++
|
||||
}
|
||||
return count, nil
|
||||
|
||||
return finished, nil
|
||||
}
|
||||
|
||||
func (svc *ItemService) ExportTSV(ctx context.Context, GID uuid.UUID) ([][]string, error) {
|
||||
items, err := svc.repo.Items.GetAll(ctx, GID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sheet := reporting.IOSheet{}
|
||||
|
||||
sheet.ReadItems(items)
|
||||
|
||||
return sheet.TSV()
|
||||
}
|
||||
|
||||
func (svc *ItemService) ExportBillOfMaterialsTSV(ctx context.Context, GID uuid.UUID) ([]byte, error) {
|
||||
items, err := svc.repo.Items.GetAll(ctx, GID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return reporting.BillOfMaterialsTSV(items)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue