# This is a combination of 2 commits.

# This is the 1st commit message:

unmarshal all formats, merge them in the single table, users are truly unique

# This is the commit message #2:

i

1 files changed, 232 insertions, 0 deletions

diff --git a/pkg/adapters/csv/csv.go b/pkg/adapters/csv/csv.go
new file mode 100644
index 0000000..1d8154a
--- /dev/null
+++ b/pkg/adapters/csv/csv.go
@@ -0,0 +1,232 @@
+package csv
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"airlines/pkg/airports"
+	"airlines/pkg/model"
+	"airlines/pkg/names"
+
+	"github.com/schollz/progressbar/v3"
+)
+
+type csvData struct {
+	data                [][]string
+	ignoreNonLatinUsers bool
+}
+
+func UnmarshallCsv(filePath string, ignoreNonLatinUsers bool) (*csvData, error) {
+	d := &csvData{ignoreNonLatinUsers: ignoreNonLatinUsers}
+
+	f, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	csvReader := csv.NewReader(f)
+	d.data, err = csvReader.ReadAll()
+	if err != nil {
+		log.Fatal("Unable to parse file as CSV for "+filePath, err)
+	}
+
+	return d, nil
+}
+
+func createUser(cols map[string]int, data []string) (*model.User, error) {
+	fio, err := names.ParseLatinName(data[cols["PaxName"]])
+	if err != nil {
+		return nil, errors.New(data[cols["PaxName"]] + ":" + err.Error())
+	}
+
+	var bdayDate time.Time
+	bday := strings.TrimSpace(data[cols["PaxBirthDate"]]) // "YYYY-MM-DD"
+	if bday == "" {
+		bdayDate = model.SentinelBirthday()
+	} else {
+		bdayDate, err = time.Parse("2006-01-02", bday)
+		if err != nil {
+			return nil, fmt.Errorf("bad birthday %q: %w", bday, err)
+		}
+	}
+
+	u := &model.User{
+		Name:        strings.ToUpper(fio.First),
+		Surname:     strings.ToUpper(fio.Last),
+		Fathersname: strings.ToUpper(fio.Patronymic),
+		Birthday:    bdayDate,
+	}
+	return u, nil
+}
+
+func createCard(cols map[string]int, data []string) (*model.Card, error) {
+	prefix, number, _ := model.ParseCardLine(data[cols["CardNumber"]])
+	if prefix == "" && number == 0 {
+		return nil, errors.New("bad card id")
+	}
+
+	return &model.Card{
+		Prefix: prefix,
+		Number: number,
+	}, nil
+}
+
+var hhmm = regexp.MustCompile(`^\d{2}:\d{2}$`)
+
+func NormalizeTime(s string) string {
+	if hhmm.MatchString(s) {
+		h := (int(s[0]-'0')*10 + int(s[1]-'0'))
+		m := (int(s[3]-'0')*10 + int(s[4]-'0'))
+		if h < 24 && m < 60 {
+			return s
+		}
+		return ""
+	}
+
+	digits := make([]byte, 0, 4)
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if c >= '0' && c <= '9' {
+			digits = append(digits, c)
+		} else if c == ':' {
+			continue
+		} else {
+			return ""
+		}
+	}
+
+	if len(digits) == 3 {
+		h := int(digits[0] - '0')
+		m := int(digits[1]-'0')*10 + int(digits[2]-'0')
+		if h < 24 && m < 60 {
+			return fmt.Sprintf("%02d:%02d", h, m)
+		}
+		return ""
+	}
+
+	if len(digits) == 4 {
+		h := int(digits[0]-'0')*10 + int(digits[1]-'0')
+		m := int(digits[2]-'0')*10 + int(digits[3]-'0')
+		if h < 24 && m < 60 {
+			return fmt.Sprintf("%02d:%02d", h, m)
+		}
+		return ""
+	}
+
+	return ""
+}
+
+func createFlight(cols map[string]int, data []string) (*model.Flight, error) {
+	f := &model.Flight{}
+
+	f.From = strings.TrimSpace(data[cols["From"]])
+	f.To = strings.TrimSpace(data[cols["Dest"]])
+	f.Number = strings.TrimSpace(data[cols["FlightCodeSh"]])
+	f.Code = strings.TrimSpace(data[cols["Code"]])
+
+	d, err := time.Parse("2006-01-02 15:04", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]])))
+	if err != nil {
+		return nil, fmt.Errorf("invalid Date %q: %w", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]])), err)
+	}
+
+	ap, _ := airports.LookupIATA(f.From)
+	f.FromCoords.Lat = ap.Latitude
+	f.FromCoords.Long = ap.Longitude
+
+	loc := model.TzFromAirportRecord(ap)
+	departLocal := time.Date(d.Year(), d.Month(), d.Day(), d.Hour(), d.Minute(), 0, 0, loc)
+	f.Date = departLocal.UTC()
+	f.HasTime = strings.TrimSpace(data[cols["DepartTime"]]) != ""
+
+	ap, _ = airports.LookupIATA(f.To)
+	f.ToCoords.Lat = ap.Latitude
+	f.ToCoords.Long = ap.Longitude
+
+	return f, nil
+}
+
+func (dat *csvData) DumpToDb(store model.Store) {
+	bar := progressbar.Default(int64(len(dat.data)-1), "dumping csv data")
+
+	cols := map[string]int{}
+
+	for i, col := range dat.data[0] {
+		cols[col] = i
+	}
+
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, 8)
+
+	for i := 1; i < len(dat.data); i++ {
+		sem <- struct{}{}
+		wg.Add(1)
+		go func() {
+			defer func() { <-sem }()
+			defer wg.Done()
+			defer bar.Add(1)
+
+			var u *model.User
+			var err error
+			if dat.ignoreNonLatinUsers {
+				goto skipUser
+			}
+			u, err = createUser(cols, dat.data[i])
+			if err != nil {
+				fmt.Println(err)
+				// return
+			}
+			u, err = store.SaveUser(u)
+			if err != nil {
+				fmt.Println(err)
+				// return
+			}
+		skipUser:
+
+			c, err := createCard(cols, dat.data[i])
+			if err != nil {
+				// fmt.Println(err)
+				// return
+			}
+			if c != nil {
+				if u != nil {
+					c.UserID = u.ID
+				}
+				c, err = store.SaveCard(c)
+				if err != nil {
+					// fmt.Println(err)
+					// return
+				}
+			}
+
+			f, err := createFlight(cols, dat.data[i])
+			if err != nil {
+				fmt.Println(err)
+				return
+			}
+			if f != nil {
+				if u != nil {
+					f.UserID = u.ID
+				}
+				if c != nil {
+					f.CardID = c.ID
+				}
+
+				_, err = store.SaveFlight(f)
+				if err != nil {
+					fmt.Println(err)
+					// return
+				}
+			}
+		}()
+
+	}
+
+}