aboutsummaryrefslogtreecommitdiff
path: root/pkg/adapters
diff options
context:
space:
mode:
Diffstat (limited to 'pkg/adapters')
-rw-r--r--pkg/adapters/csv/csv.go232
-rw-r--r--pkg/adapters/json/json.go35
-rw-r--r--pkg/adapters/json/model.go40
-rw-r--r--pkg/adapters/xlsx/model.go60
-rw-r--r--pkg/adapters/xlsx/registry.go91
-rw-r--r--pkg/adapters/xlsx/xlsx.go84
-rw-r--r--pkg/adapters/xml/registry.go51
-rw-r--r--pkg/adapters/xml/xml.go113
-rw-r--r--pkg/adapters/yaml/yaml.go180
9 files changed, 785 insertions, 101 deletions
diff --git a/pkg/adapters/csv/csv.go b/pkg/adapters/csv/csv.go
new file mode 100644
index 0000000..1d8154a
--- /dev/null
+++ b/pkg/adapters/csv/csv.go
@@ -0,0 +1,232 @@
+package csv
+
+import (
+ "encoding/csv"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "regexp"
+ "strings"
+ "sync"
+ "time"
+
+ "airlines/pkg/airports"
+ "airlines/pkg/model"
+ "airlines/pkg/names"
+
+ "github.com/schollz/progressbar/v3"
+)
+
+type csvData struct {
+ data [][]string
+ ignoreNonLatinUsers bool
+}
+
+func UnmarshallCsv(filePath string, ignoreNonLatinUsers bool) (*csvData, error) {
+ d := &csvData{ignoreNonLatinUsers: ignoreNonLatinUsers}
+
+ f, err := os.Open(filePath)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ csvReader := csv.NewReader(f)
+ d.data, err = csvReader.ReadAll()
+ if err != nil {
+ log.Fatal("Unable to parse file as CSV for "+filePath, err)
+ }
+
+ return d, nil
+}
+
+func createUser(cols map[string]int, data []string) (*model.User, error) {
+ fio, err := names.ParseLatinName(data[cols["PaxName"]])
+ if err != nil {
+ return nil, errors.New(data[cols["PaxName"]] + ":" + err.Error())
+ }
+
+ var bdayDate time.Time
+ bday := strings.TrimSpace(data[cols["PaxBirthDate"]]) // "YYYY-MM-DD"
+ if bday == "" {
+ bdayDate = model.SentinelBirthday()
+ } else {
+ bdayDate, err = time.Parse("2006-01-02", bday)
+ if err != nil {
+ return nil, fmt.Errorf("bad birthday %q: %w", bday, err)
+ }
+ }
+
+ u := &model.User{
+ Name: strings.ToUpper(fio.First),
+ Surname: strings.ToUpper(fio.Last),
+ Fathersname: strings.ToUpper(fio.Patronymic),
+ Birthday: bdayDate,
+ }
+ return u, nil
+}
+
+func createCard(cols map[string]int, data []string) (*model.Card, error) {
+ prefix, number, _ := model.ParseCardLine(data[cols["CardNumber"]])
+ if prefix == "" && number == 0 {
+ return nil, errors.New("bad card id")
+ }
+
+ return &model.Card{
+ Prefix: prefix,
+ Number: number,
+ }, nil
+}
+
+var hhmm = regexp.MustCompile(`^\d{2}:\d{2}$`)
+
+func NormalizeTime(s string) string {
+ if hhmm.MatchString(s) {
+ h := (int(s[0]-'0')*10 + int(s[1]-'0'))
+ m := (int(s[3]-'0')*10 + int(s[4]-'0'))
+ if h < 24 && m < 60 {
+ return s
+ }
+ return ""
+ }
+
+ digits := make([]byte, 0, 4)
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ if c >= '0' && c <= '9' {
+ digits = append(digits, c)
+ } else if c == ':' {
+ continue
+ } else {
+ return ""
+ }
+ }
+
+ if len(digits) == 3 {
+ h := int(digits[0] - '0')
+ m := int(digits[1]-'0')*10 + int(digits[2]-'0')
+ if h < 24 && m < 60 {
+ return fmt.Sprintf("%02d:%02d", h, m)
+ }
+ return ""
+ }
+
+ if len(digits) == 4 {
+ h := int(digits[0]-'0')*10 + int(digits[1]-'0')
+ m := int(digits[2]-'0')*10 + int(digits[3]-'0')
+ if h < 24 && m < 60 {
+ return fmt.Sprintf("%02d:%02d", h, m)
+ }
+ return ""
+ }
+
+ return ""
+}
+
+func createFlight(cols map[string]int, data []string) (*model.Flight, error) {
+ f := &model.Flight{}
+
+ f.From = strings.TrimSpace(data[cols["From"]])
+ f.To = strings.TrimSpace(data[cols["Dest"]])
+ f.Number = strings.TrimSpace(data[cols["FlightCodeSh"]])
+ f.Code = strings.TrimSpace(data[cols["Code"]])
+
+ d, err := time.Parse("2006-01-02 15:04", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]])))
+ if err != nil {
+ return nil, fmt.Errorf("invalid Date %q: %w", strings.TrimSpace(data[cols["DepartDate"]])+" "+NormalizeTime(strings.TrimSpace(data[cols["DepartTime"]])), err)
+ }
+
+ ap, _ := airports.LookupIATA(f.From)
+ f.FromCoords.Lat = ap.Latitude
+ f.FromCoords.Long = ap.Longitude
+
+ loc := model.TzFromAirportRecord(ap)
+ departLocal := time.Date(d.Year(), d.Month(), d.Day(), d.Hour(), d.Minute(), 0, 0, loc)
+ f.Date = departLocal.UTC()
+ f.HasTime = strings.TrimSpace(data[cols["DepartTime"]]) != ""
+
+ ap, _ = airports.LookupIATA(f.To)
+ f.ToCoords.Lat = ap.Latitude
+ f.ToCoords.Long = ap.Longitude
+
+ return f, nil
+}
+
+func (dat *csvData) DumpToDb(store model.Store) {
+ bar := progressbar.Default(int64(len(dat.data)-1), "dumping csv data")
+
+ cols := map[string]int{}
+
+ for i, col := range dat.data[0] {
+ cols[col] = i
+ }
+
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, 8)
+
+ for i := 1; i < len(dat.data); i++ {
+ sem <- struct{}{}
+ wg.Add(1)
+ go func() {
+ defer func() { <-sem }()
+ defer wg.Done()
+ defer bar.Add(1)
+
+ var u *model.User
+ var err error
+ if dat.ignoreNonLatinUsers {
+ goto skipUser
+ }
+ u, err = createUser(cols, dat.data[i])
+ if err != nil {
+ fmt.Println(err)
+ // return
+ }
+ u, err = store.SaveUser(u)
+ if err != nil {
+ fmt.Println(err)
+ // return
+ }
+ skipUser:
+
+ c, err := createCard(cols, dat.data[i])
+ if err != nil {
+ // fmt.Println(err)
+ // return
+ }
+ if c != nil {
+ if u != nil {
+ c.UserID = u.ID
+ }
+ c, err = store.SaveCard(c)
+ if err != nil {
+ // fmt.Println(err)
+ // return
+ }
+ }
+
+ f, err := createFlight(cols, dat.data[i])
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ if f != nil {
+ if u != nil {
+ f.UserID = u.ID
+ }
+ if c != nil {
+ f.CardID = c.ID
+ }
+
+ _, err = store.SaveFlight(f)
+ if err != nil {
+ fmt.Println(err)
+ // return
+ }
+ }
+ }()
+
+ }
+
+}
diff --git a/pkg/adapters/json/json.go b/pkg/adapters/json/json.go
index 47a563e..73d3ae8 100644
--- a/pkg/adapters/json/json.go
+++ b/pkg/adapters/json/json.go
@@ -2,14 +2,15 @@ package json
import (
"bytes"
- "context"
"encoding/json"
"fmt"
+ "os"
"strings"
"time"
"airlines/pkg/model"
- "airlines/pkg/store"
+
+ "github.com/schollz/progressbar/v3"
)
type DateYMD struct {
@@ -97,11 +98,29 @@ type JsonCard struct {
Number Trimmed `json:"Number"`
}
-func (r *JsonRoot) DumpToDb(ctx context.Context, s *store.Store) {
+func UnmarshalJsonRoot(path string) (*JsonRoot, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ panic(err)
+ }
+ defer f.Close()
+
+ dec := json.NewDecoder(f)
+ var root JsonRoot
+ if err := dec.Decode(&root); err != nil {
+ return nil, err
+ }
+ return &root, nil
+}
+
+func (r *JsonRoot) DumpToDb(store model.Store) {
var err error
- for _, user := range r.ForumProfiles {
+ bar := progressbar.Default(int64(len(r.ForumProfiles)), "dumping json")
+
+ for i, user := range r.ForumProfiles {
+ bar.Set(i)
dbUser, _ := user.ToUser()
- dbUser, err = s.CreateOrGetUser(ctx, dbUser)
+ dbUser, err = store.SaveUser(dbUser)
if err != nil {
panic(err)
}
@@ -111,7 +130,8 @@ func (r *JsonRoot) DumpToDb(ctx context.Context, s *store.Store) {
if err != nil {
panic(err)
}
- _, err = s.AddCardsToUser(ctx, dbUser.ID, dbCard)
+ dbCard.UserID = dbUser.ID
+ _, err = store.SaveCard(dbCard)
// данные говно
if err != nil {
fmt.Println(err)
@@ -121,7 +141,8 @@ func (r *JsonRoot) DumpToDb(ctx context.Context, s *store.Store) {
for _, flight := range user.RegisteredFlights {
dbFlight, _ := flight.ToFlight()
- _, err = s.AddFlightToUser(ctx, dbUser.ID, dbFlight)
+ dbFlight.UserID = dbUser.ID
+ _, err = store.SaveFlight(dbFlight)
if err != nil {
fmt.Println(err)
}
diff --git a/pkg/adapters/json/model.go b/pkg/adapters/json/model.go
index 2cc5d8e..86d7321 100644
--- a/pkg/adapters/json/model.go
+++ b/pkg/adapters/json/model.go
@@ -2,10 +2,13 @@ package json
import (
"strconv"
+ "strings"
"time"
"unicode"
"airlines/pkg/model"
+
+ "airlines/pkg/airports"
)
func sOrEmpty(p *string) string {
@@ -34,9 +37,9 @@ func onlyDigits(s string) string {
func (jp JsonProfile) ToUser() (*model.User, error) {
return &model.User{
- Name: sOrEmpty(jp.RealName.FirstName),
- Surname: sOrEmpty(jp.RealName.LastName),
- Nick: sOrEmpty(&jp.NickName),
+ Name: strings.ToUpper(sOrEmpty(jp.RealName.FirstName)),
+ Surname: strings.ToUpper(sOrEmpty(jp.RealName.LastName)),
+ Nick: strings.ToUpper(sOrEmpty(&jp.NickName)),
Fathersname: "",
Sex: jp.Sex,
Birthday: model.SentinelBirthday(),
@@ -44,16 +47,27 @@ func (jp JsonProfile) ToUser() (*model.User, error) {
}
func (jf JsonFlight) ToFlight() (*model.Flight, error) {
- return &model.Flight{
- Number: jf.Flight,
- From: jf.Departure.Airport,
- FromCity: jf.Departure.City,
- FromCountry: jf.Departure.Country,
- To: jf.Arrival.Airport,
- ToCity: jf.Arrival.City,
- ToCountry: jf.Arrival.Country,
- Date: jf.Date.ToDateUTC(),
- }, nil
+ // lookup IATA codes for lat lon
+ f := &model.Flight{
+ Number: jf.Flight,
+ From: strings.ToUpper(jf.Departure.Airport),
+ To: strings.ToUpper(jf.Arrival.Airport),
+ Date: jf.Date.ToDateUTC(),
+ }
+
+ ap, _ := airports.LookupIATA(f.From)
+ f.FromCoords.Lat = ap.Latitude
+ f.FromCoords.Long = ap.Longitude
+
+ loc := model.TzFromAirportRecord(ap)
+ departLocal := time.Date(f.Date.Year(), f.Date.Month(), f.Date.Day(), 0, 0, 0, 0, loc)
+ f.Date = departLocal.UTC()
+
+ ap, _ = airports.LookupIATA(jf.Arrival.Airport)
+ f.ToCoords.Lat = ap.Latitude
+ f.ToCoords.Long = ap.Longitude
+
+ return f, nil
}
func (jc JsonCard) ToCard() (*model.Card, error) {
diff --git a/pkg/adapters/xlsx/model.go b/pkg/adapters/xlsx/model.go
index d8c5194..79434f0 100644
--- a/pkg/adapters/xlsx/model.go
+++ b/pkg/adapters/xlsx/model.go
@@ -1,13 +1,13 @@
package xlsx
import (
+ "airlines/pkg/model"
"errors"
- "regexp"
"strconv"
"strings"
"time"
- "github.com/leonm1/airports-go"
+ "airlines/pkg/airports"
)
type Ticket struct {
@@ -16,9 +16,13 @@ type Ticket struct {
Title string
FlightNumber string
FromCity string
- ToCity string
+ FromCountry string
FromAirport string
+ FromCoords model.LatLong
+ ToCity string
+ ToCountry string
ToAirport string
+ ToCoords model.LatLong
FlightDate string // (raw, expected YYYY-MM-DD; Excel text may start with ')
FlightTime string // (raw, expected HH-MM or HH:MM; Excel text may start with ')
PNR string
@@ -87,52 +91,4 @@ func two(x int) string {
return "0" + strconv.Itoa(x)
}
return strconv.Itoa(x)
-}
-
-func parseCardLine(s string) (prefix string, number uint64, bonus string) {
- raw := strings.TrimSpace(s)
- if raw == "" {
- return "", 0, ""
- }
- // number = last run of digits
- if m := regexp.MustCompile(`(\d{3,})\D*$`).FindStringSubmatch(raw); len(m) == 2 {
- if n, err := strconv.ParseUint(m[1], 10, 64); err == nil {
- number = n
- }
- }
-
- // tokens (letters with '-', '/', apostrophes)
- tokRe := regexp.MustCompile(`[A-Za-z][A-Za-z'/-]*`)
- toks := tokRe.FindAllString(s, -1)
-
- // prefix = first 2–3 letter all-caps-ish token
- for _, t := range toks {
- u := strings.ToUpper(t)
- if len(u) >= 2 && len(u) <= 3 && regexp.MustCompile(`^[A-Z]{2,3}$`).MatchString(u) {
- prefix = u
- break
- }
- }
- // bonus = all tokens except prefix
- words := []string{}
- for _, t := range toks {
- if strings.ToUpper(t) == prefix {
- continue
- }
- words = append(words, t)
- }
- if len(words) > 0 {
- bonus = strings.Join(words, " ")
- }
- if bonus == "" && prefix != "" {
- bonus = prefix
- }
- return
-}
-
-func firstNonEmpty(a, b string) string {
- if strings.TrimSpace(a) != "" {
- return a
- }
- return b
-}
+} \ No newline at end of file
diff --git a/pkg/adapters/xlsx/registry.go b/pkg/adapters/xlsx/registry.go
index 46c395e..6b98c0b 100644
--- a/pkg/adapters/xlsx/registry.go
+++ b/pkg/adapters/xlsx/registry.go
@@ -1,69 +1,102 @@
package xlsx
import (
+ "errors"
"fmt"
+ "strconv"
"strings"
+ "time"
"airlines/pkg/model"
"airlines/pkg/names"
- "github.com/leonm1/airports-go"
+ "airlines/pkg/airports"
)
-func (t Ticket) ToUser() (model.User, error) {
+func (t Ticket) ToUser() (*model.User, error) {
fio, err := names.ParseLatinName(t.Passenger)
if err != nil {
- return model.User{}, fmt.Errorf("%v %s", t.Sheet, err.Error())
+ return nil, fmt.Errorf("%v %s", t.Sheet, err.Error())
}
sex := names.GenderFromTitle(t.Title)
- u := model.User{
+ u := &model.User{
Nick: "",
- Name: fio.First,
- Surname: fio.Last,
- Fathersname: fio.Patronymic,
+ Name: strings.ToUpper(fio.First),
+ Surname: strings.ToUpper(fio.Last),
+ Fathersname: strings.ToUpper(fio.Patronymic),
Sex: sex,
}
return u, nil
}
-func (t Ticket) ToCard() (model.Card, error) {
- prefix, number, bonus := parseCardLine(t.Card)
+func (t Ticket) ToCard() (*model.Card, error) {
+ prefix, number, bonus := model.ParseCardLine(t.Card)
if number == 0 && prefix == "" && bonus == "" {
- return model.Card{}, nil
+ return nil, errors.New("do not have card")
}
- return model.Card{
+ return &model.Card{
Prefix: prefix,
Number: number,
Bonusprogramm: "",
}, nil
}
-func (t Ticket) ToFlight() (model.Flight, error) {
- // Resolve IATA records
+func (t Ticket) ToFlight() (*model.Flight, error) {
fromIATA := strings.ToUpper(strings.TrimSpace(t.FromAirport))
toIATA := strings.ToUpper(strings.TrimSpace(t.ToAirport))
fromRec, _ := airports.LookupIATA(fromIATA)
- toRec, _ := airports.LookupIATA(toIATA)
- fromCity := firstNonEmpty(strings.TrimSpace(t.FromCity), fromRec.City)
- toCity := firstNonEmpty(strings.TrimSpace(t.ToCity), toRec.City)
+ dateStr := strings.TrimLeft(strings.TrimSpace(t.FlightDate), "'")
+ timeStr := strings.TrimLeft(strings.TrimSpace(t.FlightTime), "'")
+ timeStr = strings.ReplaceAll(timeStr, "-", ":")
- fromCountry := fromRec.Country
- toCountry := toRec.Country
- departUTC, _, err := t.DateTime()
+ if dateStr == "" || timeStr == "" {
+ return nil, errors.New("missing FlightDate or FlightTime")
+ }
+
+ hh, mm, err := parseHHMM(timeStr)
+ if err != nil {
+ return nil, err
+ }
+
+ day, err := time.Parse("2006-01-02", dateStr)
if err != nil {
- return model.Flight{}, err
+ return nil, err
}
- return model.Flight{
- Number: strings.TrimSpace(t.FlightNumber),
- From: fromIATA,
- FromCity: fromCity,
- FromCountry: fromCountry,
- To: toIATA,
- ToCity: toCity,
- ToCountry: toCountry,
- Date: departUTC,
+
+ loc := model.TzFromAirportRecord(fromRec)
+ departLocal := time.Date(day.Year(), day.Month(), day.Day(), hh, mm, 0, 0, loc)
+ departUTC := departLocal.UTC()
+
+ return &model.Flight{
+ Number: strings.TrimSpace(t.FlightNumber),
+ From: fromIATA,
+ FromCoords: t.FromCoords,
+ To: toIATA,
+ ToCoords: t.ToCoords,
+ Date: departUTC,
+ HasTime: true,
}, nil
}
+
+func parseHHMM(s string) (int, int, error) {
+ // Accept "H:MM", "HH:MM"
+ parts := strings.Split(s, ":")
+ if len(parts) != 2 {
+ return 0, 0, errors.New("invalid FlightTime, expected HH:MM")
+ }
+ hh, err := strconv.Atoi(parts[0])
+ if err != nil {
+ return 0, 0, err
+ }
+ mm, err := strconv.Atoi(parts[1])
+ if err != nil {
+ return 0, 0, err
+ }
+ if hh < 0 || hh > 23 || mm < 0 || mm > 59 {
+ return 0, 0, errors.New("invalid FlightTime range")
+ }
+ return hh, mm, nil
+}
diff --git a/pkg/adapters/xlsx/xlsx.go b/pkg/adapters/xlsx/xlsx.go
index 6ef9baa..5a8427b 100644
--- a/pkg/adapters/xlsx/xlsx.go
+++ b/pkg/adapters/xlsx/xlsx.go
@@ -2,8 +2,14 @@ package xlsx
import (
"fmt"
+ "os"
"strings"
+ "sync"
+ "airlines/pkg/airports"
+ "airlines/pkg/model"
+
+ "github.com/schollz/progressbar/v3"
"github.com/xuri/excelize/v2"
)
@@ -84,7 +90,85 @@ func UnmarshallXlsxFile(fname string) ([]Ticket, error) {
return nil, err
}
+ ap, _ := airports.LookupIATA(t.FromAirport)
+ t.FromCountry = ap.Country
+ t.FromCoords.Lat = ap.Latitude
+ t.FromCoords.Long = ap.Longitude
+
+ ap, _ = airports.LookupIATA(t.ToAirport)
+ t.ToCountry = ap.Country
+ t.ToCoords.Lat = ap.Latitude
+ t.ToCoords.Long = ap.Longitude
+
tickets = append(tickets, t)
}
return tickets, nil
}
+
+func UnmarshallXlsxFiles(baseDir string) ([]Ticket, error) {
+
+ tickets := make([]Ticket, 0)
+ items, err := os.ReadDir(baseDir)
+ if err != nil {
+ panic(err)
+ }
+
+ bar := progressbar.Default(int64(len(items)), "unmarshalling xlsx")
+
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, 8)
+ for _, item := range items {
+ if !item.IsDir() {
+ wg.Add(1)
+ sem <- struct{}{}
+ go func(name string) {
+ defer func() { <-sem }()
+ defer wg.Done()
+ // fmt.Println("Processing file:", name)
+ parsedTickets, err := UnmarshallXlsxFile(baseDir + name)
+ if err != nil {
+ panic(err)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ tickets = append(tickets, parsedTickets...)
+ bar.Add(1)
+ }(item.Name())
+ }
+ }
+ wg.Wait()
+
+ return tickets, nil
+}
+
+func DumpToDb(store model.Store, tickets []Ticket) {
+ var err error
+ bar := progressbar.Default(int64(len(tickets)), "dumping xlsx")
+ for _, ticket := range tickets {
+ bar.Add(1)
+ dbUser, _ := ticket.ToUser()
+ dbUser, err = store.SaveUser(dbUser)
+ if err != nil {
+ panic(err)
+ }
+
+ dbCard, err := ticket.ToCard()
+ if err != nil {
+ goto processflight
+ }
+ dbCard.UserID = dbUser.ID
+ _, err = store.SaveCard(dbCard)
+ // данные говно
+ if err != nil {
+ fmt.Println(err)
+ }
+ processflight:
+ dbFlight, _ := ticket.ToFlight()
+ dbFlight.UserID = dbUser.ID
+ _, err = store.SaveFlight(dbFlight)
+ if err != nil {
+ fmt.Println(err)
+ }
+ }
+}
diff --git a/pkg/adapters/xml/registry.go b/pkg/adapters/xml/registry.go
new file mode 100644
index 0000000..f7b0507
--- /dev/null
+++ b/pkg/adapters/xml/registry.go
@@ -0,0 +1,51 @@
+package xml
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ "airlines/pkg/model"
+)
+
+func (u XMLUser) ToUser() (*model.User, error) {
+ return &model.User{
+ Nick: "",
+ Name: strings.ToUpper(strings.TrimSpace(u.Name.First)),
+ Surname: strings.ToUpper(strings.TrimSpace(u.Name.Last)),
+ Fathersname: "",
+ Sex: model.SexUnknown,
+ Birthday: model.SentinelBirthday(),
+ }, nil
+}
+
+func (c XMLCard) ToCard() (*model.Card, error) {
+ prefix, number, _ := model.ParseCardLine(c.Number)
+ if prefix == "" && number == 0 && strings.TrimSpace(c.Program) == "" {
+ return nil, errors.New("bad card")
+ }
+ return &model.Card{
+ Prefix: prefix,
+ Number: number,
+ Bonusprogramm: strings.TrimSpace(c.Program),
+ }, nil
+}
+
+func (a XMLActivity) ToFlight() (*model.Flight, error) {
+ fromIATA := strings.ToUpper(strings.TrimSpace(a.Departure))
+ toIATA := strings.ToUpper(strings.TrimSpace(a.Arrival))
+
+ d, err := time.Parse("2006-01-02", strings.TrimSpace(a.Date))
+ if err != nil {
+ return nil, fmt.Errorf("invalid Date %q for flight %q: %w", a.Date, a.Code, err)
+ }
+ departUTC := time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, time.UTC)
+
+ return &model.Flight{
+ Number: strings.TrimSpace(a.Code),
+ From: fromIATA,
+ To: toIATA,
+ Date: departUTC,
+ }, nil
+}
diff --git a/pkg/adapters/xml/xml.go b/pkg/adapters/xml/xml.go
new file mode 100644
index 0000000..3350c01
--- /dev/null
+++ b/pkg/adapters/xml/xml.go
@@ -0,0 +1,113 @@
+package xml
+
+import (
+ "airlines/pkg/model"
+ "encoding/xml"
+ "fmt"
+ "os"
+
+ "github.com/schollz/progressbar/v3"
+)
+
+type PointzAggregatorUsers struct {
+ XMLName xml.Name `xml:"PointzAggregatorUsers"`
+ Users []XMLUser `xml:"user"`
+}
+
+type XMLUser struct {
+ UID string `xml:"uid,attr"`
+ Name Name `xml:"name"`
+ Cards Cards `xml:"cards"`
+}
+
+type Name struct {
+ First string `xml:"first,attr"`
+ Last string `xml:"last,attr"`
+}
+
+type Cards struct {
+ Type string `xml:"type,attr"`
+ Card []XMLCard `xml:"card"`
+}
+
+type XMLCard struct {
+ Number string `xml:"number,attr"`
+ Program string `xml:"bonusprogramm"`
+ Activities Activities `xml:"activities"`
+}
+
+type Activities struct {
+ Type string `xml:"type,attr"`
+ Activitys []XMLActivity `xml:"activity"`
+}
+
+type XMLActivity struct {
+ Type string `xml:"type,attr"`
+ Code string `xml:"Code"`
+ Date string `xml:"Date"`
+ Departure string `xml:"Departure"`
+ Arrival string `xml:"Arrival"`
+ Fare string `xml:"Fare"`
+}
+
+func UnmarshalXml(path string) (*PointzAggregatorUsers, error) {
+
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+
+ var p PointzAggregatorUsers
+ if err := xml.Unmarshal(data, &p); err != nil {
+ return nil, err
+ }
+ return &p, nil
+}
+
+func (u *PointzAggregatorUsers) DumpToDb(store model.Store) error {
+ // var err error
+ bar := progressbar.Default(int64(len(u.Users)), "dumping xml")
+
+ for _, u := range u.Users {
+ dbUser, err := u.ToUser()
+ if err != nil {
+ panic(err)
+ }
+ dbUser, err = store.SaveUser(dbUser)
+ if err != nil {
+ panic(err)
+ }
+ // fmt.Println(dbUser)
+
+
+ for _, card := range u.Cards.Card {
+ dbCard, err := card.ToCard()
+ if err != nil {
+ fmt.Println(err)
+ continue
+ }
+ dbCard.UserID = dbUser.ID
+ _, err = store.SaveCard(dbCard)
+ // данные говно
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ for _, activity := range card.Activities.Activitys {
+
+ dbFlight, err := activity.ToFlight()
+ if err != nil {
+ fmt.Println(err)
+ }
+ dbFlight.UserID = dbUser.ID
+ _, err = store.SaveFlight(dbFlight)
+ if err != nil {
+ fmt.Println(err)
+ }
+ }
+ }
+ bar.Add(1)
+ }
+
+ return nil
+}
diff --git a/pkg/adapters/yaml/yaml.go b/pkg/adapters/yaml/yaml.go
new file mode 100644
index 0000000..9a79a72
--- /dev/null
+++ b/pkg/adapters/yaml/yaml.go
@@ -0,0 +1,180 @@
+package yaml
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+ "sync"
+ "time"
+
+ "airlines/pkg/airports"
+ "airlines/pkg/model"
+
+ "github.com/schollz/progressbar/v3"
+ "gopkg.in/yaml.v3"
+)
+
+// ---------- Data model ----------
+
+type FareInfo struct {
+ Class string `yaml:"CLASS" json:"class"`
+ Fare string `yaml:"FARE" json:"fare"`
+}
+
+type Flight struct {
+ FF map[string]FareInfo `yaml:"FF" json:"ff"`
+ From string `yaml:"FROM" json:"from"`
+ Status string `yaml:"STATUS" json:"status"`
+ To string `yaml:"TO" json:"to"`
+}
+
+type Schedule struct {
+ data map[string]map[string]Flight
+}
+
+// ParseSchedule reads YAML from r into a Schedule.
+func ParseSchedule(r io.Reader) (*Schedule, error) {
+ s := &Schedule{
+ data: make(map[string]map[string]Flight),
+ }
+ dec := yaml.NewDecoder(r)
+ // dec.KnownFields(true) // enable if you want strict field checking
+ if err := dec.Decode(&s.data); err != nil {
+ return nil, err
+ }
+
+ for date, flights := range s.data {
+ if flights == nil {
+ s.data[date] = map[string]Flight{}
+ }
+ }
+ return s, nil
+}
+
+func UnmarshallYaml(path string) (*Schedule, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+
+ sched, err := ParseSchedule(f)
+ if err != nil {
+ return nil, err
+ }
+
+ // for date, flights := range sched.data {
+ // fmt.Printf("Date: %s (flights: %d)\n", date, len(flights))
+ // for flightNo, fl := range flights {
+ // fmt.Printf(" %s: %s -> %s, %s\n", flightNo, fl.From, fl.To, fl.Status)
+ // for ffKey, fi := range fl.FF {
+ // fmt.Printf(" FF %s: class=%s fare=%s\n", ffKey, fi.Class, fi.Fare)
+ // }
+ // }
+ // }
+
+ // if b, err := json.MarshalIndent(sched, "", " "); err == nil {
+ // fmt.Println("\nAs JSON:")
+ // fmt.Println(string(b))
+ // }
+ return sched, nil
+}
+
+func createFlight(num, from, to, date string, cardID uint64) (*model.Flight, error) {
+ f := &model.Flight{
+ Number: num,
+ From: from,
+ To: to,
+ }
+
+ ap, _ := airports.LookupIATA(f.From)
+ f.FromCoords.Lat = ap.Latitude
+ f.FromCoords.Long = ap.Longitude
+
+ d, err := time.Parse("2006-01-02", strings.TrimSpace(date))
+ if err != nil {
+ return nil, fmt.Errorf("invalid Date %q: %w", date, err)
+ }
+
+ loc := model.TzFromAirportRecord(ap)
+ departLocal := time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, loc)
+ f.Date = departLocal.UTC()
+
+ ap, _ = airports.LookupIATA(f.To)
+ f.ToCoords.Lat = ap.Latitude
+ f.ToCoords.Long = ap.Longitude
+
+ return f, nil
+}
+
+func createCard(info string) (*model.Card, error) {
+ prefix, number, _ := model.ParseCardLine(info)
+ if prefix == "" && number == 0 {
+ return nil, errors.New("bad card id")
+ }
+
+ return &model.Card{
+ Prefix: prefix,
+ Number: number,
+ }, nil
+}
+
+func (s *Schedule) DumpToDb(store model.Store) error {
+ // var err error
+ count := int64(0)
+ for i := range s.data {
+ count += int64(len(s.data[i]))
+ }
+
+ bar := progressbar.Default(count, "dumping yaml flights")
+
+ for date := range s.data {
+ for flightID, flightInfo := range s.data[date] {
+
+ // fmt.Printf("%+v %+v %+v\n", date, flightID, flightInfo)
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, 8)
+ for cardID := range flightInfo.FF {
+ wg.Add(1)
+ sem <- struct{}{}
+ go func() {
+ defer func() { <-sem }()
+ defer wg.Done()
+ defer bar.Add(1)
+
+ card, err := createCard(cardID)
+ if err != nil {
+ fmt.Println(err)
+ // return
+ }
+ if card != nil {
+ card, err = store.SaveCard(card)
+ if err != nil {
+ fmt.Println(err)
+ // return
+ }
+ }
+
+ f, err := createFlight(flightID, flightInfo.From, flightInfo.To, date, card.ID)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ if card != nil {
+ f.UserID = card.UserID
+ }
+ if f != nil {
+ _, err = store.SaveFlight(f)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ }
+ }()
+ }
+ wg.Wait()
+ }
+ }
+ return nil
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-06 06:58:29 +0000