feat: add tsv support for import files (#160)

* feat: add tsv support for import files

* add note in docs

backend/app/api/handlers/v1/v1_ctrl_items.go

@@ -1,7 +1,6 @@
 package v1
 
 import (
-	"encoding/csv"
 	"net/http"
 
 	"github.com/hay-kot/homebox/backend/internal/core/services"
@@ -178,8 +177,7 @@ func (ctrl *V1Controller) HandleItemsImport() server.HandlerFunc {
 			return validate.NewRequestError(err, http.StatusInternalServerError)
 		}
 
-		reader := csv.NewReader(file)
+		data, err := services.ReadCsv(file)
-		data, err := reader.ReadAll()
 		if err != nil {
 			log.Err(err).Msg("failed to read csv")
 			return validate.NewRequestError(err, http.StatusInternalServerError)

backend/internal/core/services/service_items_csv.go

@@ -1,7 +1,10 @@
 package services
 
 import (
+	"bytes"
+	"encoding/csv"
 	"errors"
+	"io"
 	"strconv"
 	"strings"
 	"time"
@@ -9,6 +12,44 @@ import (
 	"github.com/hay-kot/homebox/backend/internal/data/repo"
 )
 
+func determineSeparator(data []byte) (rune, error) {
+	// First row
+	firstRow := bytes.Split(data, []byte("\n"))[0]
+
+	// find first comma or /t
+	comma := bytes.IndexByte(firstRow, ',')
+	tab := bytes.IndexByte(firstRow, '\t')
+
+	switch {
+	case comma == -1 && tab == -1:
+		return 0, errors.New("could not determine separator")
+	case tab > comma:
+		return '\t', nil
+	default:
+		return ',', nil
+	}
+}
+
+func ReadCsv(r io.Reader) ([][]string, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	reader := csv.NewReader(bytes.NewReader(data))
+
+	// Determine separator
+	sep, err := determineSeparator(data)
+
+	if err != nil {
+		return nil, err
+	}
+
+	reader.Comma = sep
+
+	return reader.ReadAll()
+}
+
 var ErrInvalidCsv = errors.New("invalid csv")
 
 const NumOfCols = 21

backend/internal/core/services/service_items_csv_test.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"bytes"
+	_ "embed"
 	"encoding/csv"
 	"fmt"
 	"reflect"
@@ -11,17 +12,14 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const CSV_DATA = `
+//go:embed testdata/import.csv
-Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes
+var CSVData_Comma []byte
-A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,Description 1,TRUE,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021,
+
-B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,Description 2,FALSE,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,10/15/2021,,,,10/15/2021,
+//go:embed testdata/import.tsv
-C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,Description 3,TRUE,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021,
+var CSVData_Tab []byte
-D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,Description 4,FALSE,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,10/21/2020,,,,10/21/2020,
-E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,Description 5,TRUE,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,10/14/2020,,,,10/14/2020,
-F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,Description 6,FALSE,,39351,Honeywell,,Amazon,65.98,09/30/2020,,09/30/2020,,,,09/30/2020,`
 
 func loadcsv() [][]string {
-	reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA)))
+	reader := csv.NewReader(bytes.NewReader(CSVData_Comma))
 
 	records, err := reader.ReadAll()
 	if err != nil {
@@ -115,3 +113,52 @@ func Test_csvRow_getLabels(t *testing.T) {
 		})
 	}
 }
+
+func Test_determineSeparator(t *testing.T) {
+	type args struct {
+		data []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    rune
+		wantErr bool
+	}{
+		{
+			name: "comma",
+			args: args{
+				data: CSVData_Comma,
+			},
+			want:    ',',
+			wantErr: false,
+		},
+		{
+			name: "tab",
+			args: args{
+				data: CSVData_Tab,
+			},
+			want:    '\t',
+			wantErr: false,
+		},
+		{
+			name: "invalid",
+			args: args{
+				data: []byte("a;b;c"),
+			},
+			want:    0,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := determineSeparator(tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("determineSeparator() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("determineSeparator() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

backend/internal/core/services/testdata/import.csv

@@ -0,0 +1,7 @@
+Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes
+A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,Description 1,TRUE,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021,
+B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,Description 2,FALSE,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,10/15/2021,,,,10/15/2021,
+C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,Description 3,TRUE,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021,
+D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,Description 4,FALSE,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,10/21/2020,,,,10/21/2020,
+E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,Description 5,TRUE,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,10/14/2020,,,,10/14/2020,
+F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,Description 6,FALSE,,39351,Honeywell,,Amazon,65.98,09/30/2020,,09/30/2020,,,,09/30/2020,

backend/internal/core/services/testdata/import.tsv

@@ -0,0 +1,7 @@
+Import Ref	Location	Labels	Quantity	Name	Description	Insured	Serial Number	Mode Number	Manufacturer	Notes	Purchase From	Purchased Price	Purchased Time	Lifetime Warranty	Warranty Expires	Warranty Details	Sold To	Sold Price	Sold Time	Sold Notes
+A	Garage	IOT;Home Assistant; Z-Wave	1	Zooz Universal Relay ZEN17	Description 1	TRUE		ZEN17	Zooz		Amazon	39.95	10/13/2021		10/13/2021				10/13/2021	
+B	Living Room	IOT;Home Assistant; Z-Wave	1	Zooz Motion Sensor	Description 2	FALSE		ZSE18	Zooz		Amazon	29.95	10/15/2021		10/15/2021				10/15/2021	
+C	Office	IOT;Home Assistant; Z-Wave	1	Zooz 110v Power Switch	Description 3	TRUE		ZEN15	Zooz		Amazon	39.95	10/13/2021		10/13/2021				10/13/2021	
+D	Downstairs	IOT;Home Assistant; Z-Wave	1	Ecolink Z-Wave PIR Motion Sensor	Description 4	FALSE		PIRZWAVE2.5-ECO	Ecolink		Amazon	35.58	10/21/2020		10/21/2020				10/21/2020	
+E	Entry	IOT;Home Assistant; Z-Wave	1	Yale Security Touchscreen Deadbolt	Description 5	TRUE		YRD226ZW2619	Yale		Amazon	120.39	10/14/2020		10/14/2020				10/14/2020	
+F	Kitchen	IOT;Home Assistant; Z-Wave	1	Smart Rocker Light Dimmer	Description 6	FALSE		39351	Honeywell		Amazon	65.98	09/30/2020		09/30/2020				09/30/2020	

docs/docs/import-csv.md

@@ -9,6 +9,9 @@ Using the CSV import is the recommended way for adding items to the database. It
  - Currently only supports importing items, locations, and labels
  - Does not support attachments. Attachments must be uploaded after import
 
+!!! tip "File Formats"
+    The CSV import supports both CSV and TSV files. The only difference is the delimiter used. CSV files use a comma `,` as the delimiter and TSV files use a tab `\t` as the delimiter. The file extension does not matter.
+
 **Template**
 
 You can use this snippet as the headers for your CSV. Copy and paste it into your spreadsheet editor of choice and fill in the value.

frontend/pages/home.vue

@@ -100,7 +100,7 @@
 
       <form @submit.prevent="submitCsvFile">
         <div class="flex flex-col gap-2 py-6">
-          <input ref="importRef" type="file" class="hidden" accept=".csv" @change="setFile" />
+          <input ref="importRef" type="file" class="hidden" accept=".csv,.tsv" @change="setFile" />
 
           <BaseButton type="button" @click="uploadCsv">
             <Icon class="h-5 w-5 mr-2" name="mdi-upload" />