From f42a917390347354b8be4ed0bc0d8f18dd84c6ba Mon Sep 17 00:00:00 2001 From: Hayden <64056131+hay-kot@users.noreply.github.com> Date: Thu, 1 Dec 2022 18:06:47 -0900 Subject: [PATCH] feat: add tsv support for import files (#160) * feat: add tsv support for import files * add note in docs --- backend/app/api/handlers/v1/v1_ctrl_items.go | 4 +- .../core/services/service_items_csv.go | 41 ++++++++++++ .../core/services/service_items_csv_test.go | 65 ++++++++++++++++--- .../core/services/testdata/import.csv | 7 ++ .../core/services/testdata/import.tsv | 7 ++ docs/docs/import-csv.md | 3 + frontend/pages/home.vue | 2 +- 7 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 backend/internal/core/services/testdata/import.csv create mode 100644 backend/internal/core/services/testdata/import.tsv diff --git a/backend/app/api/handlers/v1/v1_ctrl_items.go b/backend/app/api/handlers/v1/v1_ctrl_items.go index f6bc5ff..1873d8b 100644 --- a/backend/app/api/handlers/v1/v1_ctrl_items.go +++ b/backend/app/api/handlers/v1/v1_ctrl_items.go @@ -1,7 +1,6 @@ package v1 import ( - "encoding/csv" "net/http" "github.com/hay-kot/homebox/backend/internal/core/services" @@ -178,8 +177,7 @@ func (ctrl *V1Controller) HandleItemsImport() server.HandlerFunc { return validate.NewRequestError(err, http.StatusInternalServerError) } - reader := csv.NewReader(file) - data, err := reader.ReadAll() + data, err := services.ReadCsv(file) if err != nil { log.Err(err).Msg("failed to read csv") return validate.NewRequestError(err, http.StatusInternalServerError) diff --git a/backend/internal/core/services/service_items_csv.go b/backend/internal/core/services/service_items_csv.go index c9748f7..fb5e36a 100644 --- a/backend/internal/core/services/service_items_csv.go +++ b/backend/internal/core/services/service_items_csv.go @@ -1,7 +1,10 @@ package services import ( + "bytes" + "encoding/csv" "errors" + "io" "strconv" "strings" "time" @@ -9,6 +12,44 @@ import ( "github.com/hay-kot/homebox/backend/internal/data/repo" ) +func determineSeparator(data []byte) (rune, error) { + // First row + firstRow := bytes.Split(data, []byte("\n"))[0] + + // find first comma or /t + comma := bytes.IndexByte(firstRow, ',') + tab := bytes.IndexByte(firstRow, '\t') + + switch { + case comma == -1 && tab == -1: + return 0, errors.New("could not determine separator") + case tab > comma: + return '\t', nil + default: + return ',', nil + } +} + +func ReadCsv(r io.Reader) ([][]string, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + reader := csv.NewReader(bytes.NewReader(data)) + + // Determine separator + sep, err := determineSeparator(data) + + if err != nil { + return nil, err + } + + reader.Comma = sep + + return reader.ReadAll() +} + var ErrInvalidCsv = errors.New("invalid csv") const NumOfCols = 21 diff --git a/backend/internal/core/services/service_items_csv_test.go b/backend/internal/core/services/service_items_csv_test.go index b5b488c..675a1a9 100644 --- a/backend/internal/core/services/service_items_csv_test.go +++ b/backend/internal/core/services/service_items_csv_test.go @@ -2,6 +2,7 @@ package services import ( "bytes" + _ "embed" "encoding/csv" "fmt" "reflect" @@ -11,17 +12,14 @@ import ( "github.com/stretchr/testify/assert" ) -const CSV_DATA = ` -Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes -A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,Description 1,TRUE,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021, -B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,Description 2,FALSE,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,10/15/2021,,,,10/15/2021, -C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,Description 3,TRUE,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021, -D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,Description 4,FALSE,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,10/21/2020,,,,10/21/2020, -E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,Description 5,TRUE,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,10/14/2020,,,,10/14/2020, -F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,Description 6,FALSE,,39351,Honeywell,,Amazon,65.98,09/30/2020,,09/30/2020,,,,09/30/2020,` +//go:embed testdata/import.csv +var CSVData_Comma []byte + +//go:embed testdata/import.tsv +var CSVData_Tab []byte func loadcsv() [][]string { - reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA))) + reader := csv.NewReader(bytes.NewReader(CSVData_Comma)) records, err := reader.ReadAll() if err != nil { @@ -115,3 +113,52 @@ func Test_csvRow_getLabels(t *testing.T) { }) } } + +func Test_determineSeparator(t *testing.T) { + type args struct { + data []byte + } + tests := []struct { + name string + args args + want rune + wantErr bool + }{ + { + name: "comma", + args: args{ + data: CSVData_Comma, + }, + want: ',', + wantErr: false, + }, + { + name: "tab", + args: args{ + data: CSVData_Tab, + }, + want: '\t', + wantErr: false, + }, + { + name: "invalid", + args: args{ + data: []byte("a;b;c"), + }, + want: 0, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := determineSeparator(tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("determineSeparator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("determineSeparator() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/internal/core/services/testdata/import.csv b/backend/internal/core/services/testdata/import.csv new file mode 100644 index 0000000..08bd9c8 --- /dev/null +++ b/backend/internal/core/services/testdata/import.csv @@ -0,0 +1,7 @@ +Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes +A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,Description 1,TRUE,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021, +B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,Description 2,FALSE,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,10/15/2021,,,,10/15/2021, +C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,Description 3,TRUE,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,10/13/2021,,,,10/13/2021, +D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,Description 4,FALSE,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,10/21/2020,,,,10/21/2020, +E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,Description 5,TRUE,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,10/14/2020,,,,10/14/2020, +F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,Description 6,FALSE,,39351,Honeywell,,Amazon,65.98,09/30/2020,,09/30/2020,,,,09/30/2020, \ No newline at end of file diff --git a/backend/internal/core/services/testdata/import.tsv b/backend/internal/core/services/testdata/import.tsv new file mode 100644 index 0000000..503c777 --- /dev/null +++ b/backend/internal/core/services/testdata/import.tsv @@ -0,0 +1,7 @@ +Import Ref Location Labels Quantity Name Description Insured Serial Number Mode Number Manufacturer Notes Purchase From Purchased Price Purchased Time Lifetime Warranty Warranty Expires Warranty Details Sold To Sold Price Sold Time Sold Notes +A Garage IOT;Home Assistant; Z-Wave 1 Zooz Universal Relay ZEN17 Description 1 TRUE ZEN17 Zooz Amazon 39.95 10/13/2021 10/13/2021 10/13/2021 +B Living Room IOT;Home Assistant; Z-Wave 1 Zooz Motion Sensor Description 2 FALSE ZSE18 Zooz Amazon 29.95 10/15/2021 10/15/2021 10/15/2021 +C Office IOT;Home Assistant; Z-Wave 1 Zooz 110v Power Switch Description 3 TRUE ZEN15 Zooz Amazon 39.95 10/13/2021 10/13/2021 10/13/2021 +D Downstairs IOT;Home Assistant; Z-Wave 1 Ecolink Z-Wave PIR Motion Sensor Description 4 FALSE PIRZWAVE2.5-ECO Ecolink Amazon 35.58 10/21/2020 10/21/2020 10/21/2020 +E Entry IOT;Home Assistant; Z-Wave 1 Yale Security Touchscreen Deadbolt Description 5 TRUE YRD226ZW2619 Yale Amazon 120.39 10/14/2020 10/14/2020 10/14/2020 +F Kitchen IOT;Home Assistant; Z-Wave 1 Smart Rocker Light Dimmer Description 6 FALSE 39351 Honeywell Amazon 65.98 09/30/2020 09/30/2020 09/30/2020 \ No newline at end of file diff --git a/docs/docs/import-csv.md b/docs/docs/import-csv.md index 67b1e1d..f289143 100644 --- a/docs/docs/import-csv.md +++ b/docs/docs/import-csv.md @@ -9,6 +9,9 @@ Using the CSV import is the recommended way for adding items to the database. It - Currently only supports importing items, locations, and labels - Does not support attachments. Attachments must be uploaded after import +!!! tip "File Formats" + The CSV import supports both CSV and TSV files. The only difference is the delimiter used. CSV files use a comma `,` as the delimiter and TSV files use a tab `\t` as the delimiter. The file extension does not matter. + **Template** You can use this snippet as the headers for your CSV. Copy and paste it into your spreadsheet editor of choice and fill in the value. diff --git a/frontend/pages/home.vue b/frontend/pages/home.vue index 0113a0b..7723d95 100644 --- a/frontend/pages/home.vue +++ b/frontend/pages/home.vue @@ -100,7 +100,7 @@
- +