refactor: csv import process

cmd/web/handlers:equipment:import.go

@@ -15,10 +15,8 @@
 package main
 
 import (
-	"fmt"
 	"net/http"
 	"net/url"
-	"strings"
 
 	"github.com/bit8bytes/gearberg/internal/equipmentimports"
 	"github.com/bit8bytes/gearberg/internal/httperr"
@@ -30,91 +28,14 @@ type equipmentImportData struct {
 	Error string
 }
 
-// importPreviewRow is a display-oriented view of a staged import row.
-// Serialized rows with the same name are collapsed into a single entry;
-// Stock holds the unit count for serialized items and quantity for bulk.
-type importPreviewRow struct {
-	RowNumber    int64
-	Name         string
-	TypeLabel    string
-	CategoryName string
-	Stock        string
-	Status       string
-	ErrorMessage string
-}
-
 type equipmentImportPreviewData struct {
 	OrgID      string
 	ImportID   string
-	Rows       []importPreviewRow
+	Rows       []equipmentimports.GroupedRow
 	CountNew   int
 	CountError int
 }
 
-// groupImportRows collapses serialized staging rows that share a name into one
-// preview row and returns per-item counts for new and error items.
-func groupImportRows(staged []equipmentimports.Row) (rows []importPreviewRow, cntNew, cntError int) {
-	type group struct {
-		row    importPreviewRow
-		total  int
-		hasErr bool
-	}
-	seen := make(map[string]*group)
-	var order []string
-
-	for _, r := range staged {
-		if !strings.EqualFold(r.TypeLabel, "serialized") {
-			pr := importPreviewRow{
-				RowNumber:    r.RowNumber,
-				Name:         r.Name,
-				TypeLabel:    r.TypeLabel,
-				CategoryName: r.CategoryName,
-				Stock:        r.Quantity,
-				Status:       r.Status,
-				ErrorMessage: r.ErrorMessage,
-			}
-			rows = append(rows, pr)
-			if r.Status == equipmentimports.StatusNew {
-				cntNew++
-			} else {
-				cntError++
-			}
-			continue
-		}
-
-		key := strings.ToLower(r.Name)
-		if _, ok := seen[key]; !ok {
-			seen[key] = &group{row: importPreviewRow{
-				RowNumber:    r.RowNumber,
-				Name:         r.Name,
-				TypeLabel:    r.TypeLabel,
-				CategoryName: r.CategoryName,
-				Status:       equipmentimports.StatusNew,
-			}}
-			order = append(order, key)
-		}
-		g := seen[key]
-		g.total++
-		if r.Status == equipmentimports.StatusError && !g.hasErr {
-			g.hasErr = true
-			g.row.Status = equipmentimports.StatusError
-			g.row.ErrorMessage = r.ErrorMessage
-		}
-	}
-
-	for _, key := range order {
-		g := seen[key]
-		g.row.Stock = fmt.Sprintf("%d", g.total)
-		rows = append(rows, g.row)
-		if g.hasErr {
-			cntError++
-		} else {
-			cntNew++
-		}
-	}
-	return
-}
-
 // getEquipmentImport serves the upload form when no ?id= param is present,
 // or the staging preview when ?id= is set (after a successful upload).
 func (app *application) getEquipmentImport(w http.ResponseWriter, r *http.Request) *httperr.Error {
@@ -171,7 +92,7 @@ func (app *application) renderImportPreview(w http.ResponseWriter, r *http.Reque
 		return httperr.InternalServerError(err)
 	}
 
-	previewRows, cntNew, cntError := groupImportRows(staged)
+	previewRows, cntNew, cntError := equipmentimports.GroupRows(staged)
 
 	data := app.html.TemplateData(r)
 	data.Data = equipmentImportPreviewData{

cmd/web/handlers:equipment:import_test.go

@@ -66,7 +66,7 @@ func assertRoundTrip(t *testing.T, body []byte) {
 	assertSonySerials(t, rows)
 }
 
-func parseExportCSV(t *testing.T, body []byte) []equipmentimports.RawRow {
+func parseExportCSV(t *testing.T, body []byte) []equipmentimports.ProcessedRow {
 	t.Helper()
 	// Strip the UTF-8 BOM the handler prepends so ParseCSV sees clean bytes.
 	if len(body) >= 3 && body[0] == 0xEF && body[1] == 0xBB && body[2] == 0xBF {
@@ -79,11 +79,11 @@ func parseExportCSV(t *testing.T, body []byte) []equipmentimports.RawRow {
 	return rows
 }
 
-func assertExportCounts(t *testing.T, rows []equipmentimports.RawRow) {
+func assertExportCounts(t *testing.T, rows []equipmentimports.ProcessedRow) {
 	t.Helper()
 	counts := make(map[string]int)
 	for _, r := range rows {
-		counts[r.Name]++
+		counts[r.Data.Name]++
 	}
 	if counts["Shure SM58"] != 1 {
 		t.Errorf("Shure SM58: want 1 export row, got %d", counts["Shure SM58"])
@@ -96,21 +96,21 @@ func assertExportCounts(t *testing.T, rows []equipmentimports.RawRow) {
 	}
 }
 
-func assertShureQuantity(t *testing.T, rows []equipmentimports.RawRow) {
+func assertShureQuantity(t *testing.T, rows []equipmentimports.ProcessedRow) {
 	t.Helper()
 	for _, r := range rows {
-		if r.Name == "Shure SM58" && r.Quantity != "7" {
-			t.Errorf("Shure SM58: want quantity 7, got %q", r.Quantity)
+		if r.Data.Name == "Shure SM58" && r.Data.Quantity != "7" {
+			t.Errorf("Shure SM58: want quantity 7, got %q", r.Data.Quantity)
 		}
 	}
 }
 
-func assertSonySerials(t *testing.T, rows []equipmentimports.RawRow) {
+func assertSonySerials(t *testing.T, rows []equipmentimports.ProcessedRow) {
 	t.Helper()
 	serials := make(map[string]bool)
 	for _, r := range rows {
-		if r.Name == "Sony A7 IV" {
-			serials[r.UnitSerialNumber] = true
+		if r.Data.Name == "Sony A7 IV" {
+			serials[r.Data.UnitSerialNumber] = true
 		}
 	}
 	for _, want := range []string{"SN-A7IV-001", "SN-A7IV-002"} {

internal/equipmentimports/csv.go

@@ -13,61 +13,42 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-// Package equipmentimports provides imports functionality.
 package equipmentimports
 
 import (
-	"bufio"
-	"encoding/csv"
+	"context"
 	"fmt"
 	"io"
 	"strings"
-)
-
-// ParseCSV reads a CSV (with or without a UTF-8 BOM) and returns the data rows.
-// The header row must match ExpectedHeaders exactly.
-func ParseCSV(r io.Reader) ([]RawRow, error) {
-	br := bufio.NewReader(r)
-	// Strip UTF-8 BOM produced by the export so round-tripped files parse cleanly.
-	if peek, err := br.Peek(3); err == nil && peek[0] == 0xEF && peek[1] == 0xBB && peek[2] == 0xBF {
-		_, _ = br.Discard(3)
-	}
-	cr := csv.NewReader(br)
-	cr.TrimLeadingSpace = true
-	cr.FieldsPerRecord = -1 // allow variable field counts; short rows are padded in readRows
 
-	header, err := cr.Read()
-	if err != nil {
-		return nil, fmt.Errorf("ParseCSV: read header: %w", err)
-	}
-	if err := validateHeader(header); err != nil {
-		return nil, err
-	}
-	return readRows(cr)
-}
+	pkgcsv "github.com/bit8bytes/gearberg/pkg/csv"
+)
 
 // columnAliases maps legacy column names to their current canonical name.
 // Old exports that used a different name for a column are accepted transparently.
 var columnAliases = map[string]string{
 	// "Has Content" was a boolean column (TRUE/FALSE) replaced by "Equipment Type"
-	// (Standard/Kit). Values are normalised in readRows.
+	// (Standard/Kit). Values are normalised in MapRecords.
 	"Has Content": "Equipment Type",
 }
 
-func validateHeader(header []string) error {
-	if len(header) != len(ExpectedHeaders) {
-		return fmt.Errorf("expected %d columns, got %d", len(ExpectedHeaders), len(header))
+// ParseCSV reads a CSV (with or without a UTF-8 BOM) and returns processed rows
+// with all values already converted to DB units (cents, grams, millimetres, etc.).
+// All columns in ExpectedHeaders must be present; order and extra columns are ignored.
+// Every row is initialised to StateValid; ImportID and OrgID are left empty —
+// Stage sets them before persisting.
+func ParseCSV(r io.Reader) ([]ProcessedRow, error) {
+	rd := &pkgcsv.Reader{Aliases: columnAliases}
+	records, err := rd.Read(context.Background(), r)
+	if err != nil {
+		return nil, fmt.Errorf("ParseCSV: %w", err)
 	}
-	for i, h := range header {
-		canonical := h
-		if alias, ok := columnAliases[h]; ok {
-			canonical = alias
-		}
-		if canonical != ExpectedHeaders[i] {
-			return fmt.Errorf("column %d: expected %q, got %q", i+1, ExpectedHeaders[i], h)
+	for _, name := range ExpectedHeaders {
+		if _, ok := records[0].Fields[name]; !ok {
+			return nil, fmt.Errorf("ParseCSV: missing required column %q", name)
 		}
 	}
-	return nil
+	return MapRecords(records, "", ""), nil
 }
 
 // normalizeEquipmentTypeLabel maps legacy boolean values from the old "Has Content"
@@ -82,55 +63,3 @@ func normalizeEquipmentTypeLabel(v string) string {
 		return v
 	}
 }
-
-// readRows reads data rows after the header has been consumed.
-// Column positions must match ExpectedHeaders exactly.
-func readRows(cr *csv.Reader) ([]RawRow, error) {
-	var rows []RawRow
-	for {
-		record, err := cr.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, fmt.Errorf("readRows: %w", err)
-		}
-		if len(record) < len(ExpectedHeaders) {
-			padded := make([]string, len(ExpectedHeaders))
-			copy(padded, record)
-			record = padded
-		}
-		rows = append(rows, RawRow{
-			Name:                   strings.TrimSpace(record[0]),
-			TypeLabel:              strings.TrimSpace(record[1]),
-			UsageTypeLabel:         strings.TrimSpace(record[2]),
-			CategoryName:           strings.TrimSpace(record[3]),
-			ManufacturerName:       strings.TrimSpace(record[4]),
-			LocationName:           strings.TrimSpace(record[5]),
-			RentalPrice:            strings.TrimSpace(record[6]),
-			ResalePrice:            strings.TrimSpace(record[7]),
-			Notes:                  strings.TrimSpace(record[8]),
-			WeightG:                strings.TrimSpace(record[9]),
-			WidthMm:                strings.TrimSpace(record[10]),
-			HeightMm:               strings.TrimSpace(record[11]),
-			DepthMm:                strings.TrimSpace(record[12]),
-			VoltageV:               strings.TrimSpace(record[13]),
-			CurrentA:               strings.TrimSpace(record[14]),
-			PowerW:                 strings.TrimSpace(record[15]),
-			WireGaugeMM2X100:       strings.TrimSpace(record[16]),
-			Quantity:               strings.TrimSpace(record[17]),
-			EquipmentTypeLabel:     normalizeEquipmentTypeLabel(record[18]),
-			UnitSerialNumber:       strings.TrimSpace(record[19]),
-			UnitManufacturerSerial: strings.TrimSpace(record[20]),
-			UnitPurchasePrice:      strings.TrimSpace(record[21]),
-			UnitPurchasedAt:        strings.TrimSpace(record[22]),
-			NextInspectionAt:       strings.TrimSpace(record[23]),
-			UnitIsActive:           strings.TrimSpace(record[24]),
-			UnitRemark:             strings.TrimSpace(record[25]),
-		})
-	}
-	if len(rows) == 0 {
-		return nil, fmt.Errorf("readRows: no data rows")
-	}
-	return rows, nil
-}