package csv import ( "encoding/csv" "errors" "fmt" "log" "os" "regexp" "strings" "sync" "time" "airlines/pkg/airports" "airlines/pkg/model" "airlines/pkg/names" "github.com/schollz/progressbar/v3" ) type csvData struct { data [][]string ignoreNonLatinUsers bool } func UnmarshallCsv(filePath string, ignoreNonLatinUsers bool) (*csvData, error) { d := &csvData{ignoreNonLatinUsers: ignoreNonLatinUsers} f, err := os.Open(filePath) if err != nil { return nil, err } defer f.Close() csvReader := csv.NewReader(f) d.data, err = csvReader.ReadAll() if err != nil { log.Fatal("Unable to parse file as CSV for "+filePath, err) } return d, nil } func createUser(cols map[string]int, data []string) (*model.User, error) { fio, err := names.ParseLatinName(data[cols["PaxName"]]) if err != nil { return nil, errors.New(data[cols["PaxName"]] + ":" + err.Error()) } var bdayDate time.Time bday := strings.TrimSpace(data[cols["PaxBirthDate"]]) // "YYYY-MM-DD" if bday == "" { bdayDate = model.SentinelBirthday() } else { bdayDate, err = time.Parse("2006-01-02", bday) if err != nil { return nil, fmt.Errorf("bad birthday %q: %w", bday, err) } } u := &model.User{ Name: strings.ToUpper(fio.First), Surname: strings.ToUpper(fio.Last), Fathersname: strings.ToUpper(fio.Patronymic), Birthday: bdayDate, } return u, nil } func createCard(cols map[string]int, data []string) (*model.Card, error) { prefix, number, _ := model.ParseCardLine(data[cols["CardNumber"]]) if prefix == "" && number == 0 { return nil, errors.New("bad card id") } return &model.Card{ Prefix: prefix, Number: number, }, nil } var hhmm = regexp.MustCompile(`^\d{2}:\d{2}$`) func NormalizeTime(s string) string { if hhmm.MatchString(s) { h := (int(s[0]-'0')*10 + int(s[1]-'0')) m := (int(s[3]-'0')*10 + int(s[4]-'0')) if h < 24 && m < 60 { return s } return "" } digits := make([]byte, 0, 4) for i := 0; i < len(s); i++ { c := s[i] if c >= '0' && c <= '9' { digits = append(digits, c) } else if c == ':' { continue } else { return "" } } if len(digits) == 3 { h := int(digits[0] - '0') m := int(digits[1]-'0')*10 + int(digits[2]-'0') if h < 24 && m < 60 { return fmt.Sprintf("%02d:%02d", h, m) } return "" } if len(digits) == 4 { h := int(digits[0]-'0')*10 + int(digits[1]-'0') m := int(digits[2]-'0')*10 + int(digits[3]-'0') if h < 24 && m < 60 { return fmt.Sprintf("%02d:%02d", h, m) } return "" } return "" } func createFlight(cols map[string]int, data []string) (*model.Flight, error) { f := &model.Flight{} f.From = strings.TrimSpace(data[cols["From"]]) f.To = strings.TrimSpace(data[cols["Dest"]]) f.Number = strings.TrimSpace(data[cols["FlightCodeSh"]]) f.Code = strings.TrimSpace(data[cols["Code"]]) d, err := time.Parse("2006-01-02 15:04", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]]))) if err != nil { return nil, fmt.Errorf("invalid Date %q: %w", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]])), err) } ap, _ := airports.LookupIATA(f.From) f.FromCoords.Lat = ap.Latitude f.FromCoords.Long = ap.Longitude loc := model.TzFromAirportRecord(ap) departLocal := time.Date(d.Year(), d.Month(), d.Day(), d.Hour(), d.Minute(), 0, 0, loc) f.Date = departLocal.UTC() f.HasTime = strings.TrimSpace(data[cols["DepartTime"]]) != "" ap, _ = airports.LookupIATA(f.To) f.ToCoords.Lat = ap.Latitude f.ToCoords.Long = ap.Longitude return f, nil } func (dat *csvData) DumpToDb(store model.Store) { bar := progressbar.Default(int64(len(dat.data)-1), "dumping csv data") cols := map[string]int{} for i, col := range dat.data[0] { cols[col] = i } var wg sync.WaitGroup sem := make(chan struct{}, 8) for i := 1; i < len(dat.data); i++ { sem <- struct{}{} wg.Add(1) go func() { defer func() { <-sem }() defer wg.Done() defer bar.Add(1) var u *model.User var err error if dat.ignoreNonLatinUsers { goto skipUser } u, err = createUser(cols, dat.data[i]) if err != nil { fmt.Println(err) // return } u, err = store.SaveUser(u) if err != nil { fmt.Println(err) // return } skipUser: c, err := createCard(cols, dat.data[i]) if err != nil { // fmt.Println(err) // return } if c != nil { if u != nil { c.UserID = u.ID } c, err = store.SaveCard(c) if err != nil { // fmt.Println(err) // return } } f, err := createFlight(cols, dat.data[i]) if err != nil { fmt.Println(err) return } if f != nil { if u != nil { f.UserID = u.ID } if c != nil { f.CardID = c.ID } _, err = store.SaveFlight(f) if err != nil { fmt.Println(err) // return } } }() } }