diff options
Diffstat (limited to 'pkg/adapters/csv')
| -rw-r--r-- | pkg/adapters/csv/csv.go | 232 |
1 files changed, 232 insertions, 0 deletions
diff --git a/pkg/adapters/csv/csv.go b/pkg/adapters/csv/csv.go new file mode 100644 index 0000000..1d8154a --- /dev/null +++ b/pkg/adapters/csv/csv.go @@ -0,0 +1,232 @@ +package csv + +import ( + "encoding/csv" + "errors" + "fmt" + "log" + "os" + "regexp" + "strings" + "sync" + "time" + + "airlines/pkg/airports" + "airlines/pkg/model" + "airlines/pkg/names" + + "github.com/schollz/progressbar/v3" +) + +type csvData struct { + data [][]string + ignoreNonLatinUsers bool +} + +func UnmarshallCsv(filePath string, ignoreNonLatinUsers bool) (*csvData, error) { + d := &csvData{ignoreNonLatinUsers: ignoreNonLatinUsers} + + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + csvReader := csv.NewReader(f) + d.data, err = csvReader.ReadAll() + if err != nil { + log.Fatal("Unable to parse file as CSV for "+filePath, err) + } + + return d, nil +} + +func createUser(cols map[string]int, data []string) (*model.User, error) { + fio, err := names.ParseLatinName(data[cols["PaxName"]]) + if err != nil { + return nil, errors.New(data[cols["PaxName"]] + ":" + err.Error()) + } + + var bdayDate time.Time + bday := strings.TrimSpace(data[cols["PaxBirthDate"]]) // "YYYY-MM-DD" + if bday == "" { + bdayDate = model.SentinelBirthday() + } else { + bdayDate, err = time.Parse("2006-01-02", bday) + if err != nil { + return nil, fmt.Errorf("bad birthday %q: %w", bday, err) + } + } + + u := &model.User{ + Name: strings.ToUpper(fio.First), + Surname: strings.ToUpper(fio.Last), + Fathersname: strings.ToUpper(fio.Patronymic), + Birthday: bdayDate, + } + return u, nil +} + +func createCard(cols map[string]int, data []string) (*model.Card, error) { + prefix, number, _ := model.ParseCardLine(data[cols["CardNumber"]]) + if prefix == "" && number == 0 { + return nil, errors.New("bad card id") + } + + return &model.Card{ + Prefix: prefix, + Number: number, + }, nil +} + +var hhmm = regexp.MustCompile(`^\d{2}:\d{2}$`) + +func NormalizeTime(s string) string { + if hhmm.MatchString(s) { + h := (int(s[0]-'0')*10 + int(s[1]-'0')) + m := (int(s[3]-'0')*10 + int(s[4]-'0')) + if h < 24 && m < 60 { + return s + } + return "" + } + + digits := make([]byte, 0, 4) + for i := 0; i < len(s); i++ { + c := s[i] + if c >= '0' && c <= '9' { + digits = append(digits, c) + } else if c == ':' { + continue + } else { + return "" + } + } + + if len(digits) == 3 { + h := int(digits[0] - '0') + m := int(digits[1]-'0')*10 + int(digits[2]-'0') + if h < 24 && m < 60 { + return fmt.Sprintf("%02d:%02d", h, m) + } + return "" + } + + if len(digits) == 4 { + h := int(digits[0]-'0')*10 + int(digits[1]-'0') + m := int(digits[2]-'0')*10 + int(digits[3]-'0') + if h < 24 && m < 60 { + return fmt.Sprintf("%02d:%02d", h, m) + } + return "" + } + + return "" +} + +func createFlight(cols map[string]int, data []string) (*model.Flight, error) { + f := &model.Flight{} + + f.From = strings.TrimSpace(data[cols["From"]]) + f.To = strings.TrimSpace(data[cols["Dest"]]) + f.Number = strings.TrimSpace(data[cols["FlightCodeSh"]]) + f.Code = strings.TrimSpace(data[cols["Code"]]) + + d, err := time.Parse("2006-01-02 15:04", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]]))) + if err != nil { + return nil, fmt.Errorf("invalid Date %q: %w", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]])), err) + } + + ap, _ := airports.LookupIATA(f.From) + f.FromCoords.Lat = ap.Latitude + f.FromCoords.Long = ap.Longitude + + loc := model.TzFromAirportRecord(ap) + departLocal := time.Date(d.Year(), d.Month(), d.Day(), d.Hour(), d.Minute(), 0, 0, loc) + f.Date = departLocal.UTC() + f.HasTime = strings.TrimSpace(data[cols["DepartTime"]]) != "" + + ap, _ = airports.LookupIATA(f.To) + f.ToCoords.Lat = ap.Latitude + f.ToCoords.Long = ap.Longitude + + return f, nil +} + +func (dat *csvData) DumpToDb(store model.Store) { + bar := progressbar.Default(int64(len(dat.data)-1), "dumping csv data") + + cols := map[string]int{} + + for i, col := range dat.data[0] { + cols[col] = i + } + + var wg sync.WaitGroup + sem := make(chan struct{}, 8) + + for i := 1; i < len(dat.data); i++ { + sem <- struct{}{} + wg.Add(1) + go func() { + defer func() { <-sem }() + defer wg.Done() + defer bar.Add(1) + + var u *model.User + var err error + if dat.ignoreNonLatinUsers { + goto skipUser + } + u, err = createUser(cols, dat.data[i]) + if err != nil { + fmt.Println(err) + // return + } + u, err = store.SaveUser(u) + if err != nil { + fmt.Println(err) + // return + } + skipUser: + + c, err := createCard(cols, dat.data[i]) + if err != nil { + // fmt.Println(err) + // return + } + if c != nil { + if u != nil { + c.UserID = u.ID + } + c, err = store.SaveCard(c) + if err != nil { + // fmt.Println(err) + // return + } + } + + f, err := createFlight(cols, dat.data[i]) + if err != nil { + fmt.Println(err) + return + } + if f != nil { + if u != nil { + f.UserID = u.ID + } + if c != nil { + f.CardID = c.ID + } + + _, err = store.SaveFlight(f) + if err != nil { + fmt.Println(err) + // return + } + } + }() + + } + +} |