From ded279a489631651943b5b65cdb3acb6764cf288 Mon Sep 17 00:00:00 2001 From: leshe4ka46 Date: Tue, 28 Oct 2025 13:42:55 +0300 Subject: unmarshal all formats, merge them in the single table, users are truly unique --- Catch the spies.pptx | Bin 0 -> 2928150 bytes cmd/analytics/analytics.go | 18 -- pkg/adapters/xlsx/model.go | 12 +- pkg/adapters/yaml/yaml.go | 5 +- pkg/localstore/import.go | 484 --------------------------------------------- pkg/localstore/store.go | 155 ++++----------- pkg/model/card.go | 5 +- pkg/model/model.go | 7 - pkg/model/types.go | 2 +- pkg/names/fio.go | 32 ++- pkg/names/gender.go | 12 +- pkg/store/db.go | 44 ++--- 12 files changed, 75 insertions(+), 701 deletions(-) create mode 100644 Catch the spies.pptx delete mode 100644 cmd/analytics/analytics.go delete mode 100644 pkg/localstore/import.go delete mode 100644 pkg/model/model.go diff --git a/Catch the spies.pptx b/Catch the spies.pptx new file mode 100644 index 0000000..4eb602b Binary files /dev/null and b/Catch the spies.pptx differ diff --git a/cmd/analytics/analytics.go b/cmd/analytics/analytics.go deleted file mode 100644 index e2cbdb9..0000000 --- a/cmd/analytics/analytics.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "airlines/pkg/localstore" - "fmt" -) - - -func main() { - loc := localstore.NewLocalStore() - - loc.ImportAllCSVs("/tmp/ds") - - fmt.Println(loc.FindCard("FF", 0, "")); - - - loc.Analytics() -} diff --git a/pkg/adapters/xlsx/model.go b/pkg/adapters/xlsx/model.go index 79434f0..ff92def 100644 --- a/pkg/adapters/xlsx/model.go +++ b/pkg/adapters/xlsx/model.go @@ -23,11 +23,11 @@ type Ticket struct { ToCountry string ToAirport string ToCoords model.LatLong - FlightDate string // (raw, expected YYYY-MM-DD; Excel text may start with ') - FlightTime string // (raw, expected HH-MM or HH:MM; Excel text may start with ') + FlightDate string // expected YYYY-MM-DD + FlightTime string // expected HH-MM or HH:MM PNR string Card string - TicketNumber string // (may have a leading ' in Excel) + TicketNumber string } func (t Ticket) DateTime() (time.Time, *time.Location, error) { @@ -62,13 +62,13 @@ func iataToLocation(code string) *time.Location { if err != nil { return nil } - // Prefer IANA tz name + // prefer IATA tz name if tz := strings.TrimSpace(ap.Tz); tz != "" && tz != `\N` { if loc, err := time.LoadLocation(tz); err == nil { return loc } } - // Fallback: fixed offset (no DST) + // fallback to fixed offset (no DST) if ap.Timezone != 0 { sec := int(ap.Timezone * 3600.0) return time.FixedZone("UTC"+offsetLabel(sec), sec) @@ -91,4 +91,4 @@ func two(x int) string { return "0" + strconv.Itoa(x) } return strconv.Itoa(x) -} \ No newline at end of file +} diff --git a/pkg/adapters/yaml/yaml.go b/pkg/adapters/yaml/yaml.go index 9a79a72..9304c8f 100644 --- a/pkg/adapters/yaml/yaml.go +++ b/pkg/adapters/yaml/yaml.go @@ -16,8 +16,6 @@ import ( "gopkg.in/yaml.v3" ) -// ---------- Data model ---------- - type FareInfo struct { Class string `yaml:"CLASS" json:"class"` Fare string `yaml:"FARE" json:"fare"` @@ -34,13 +32,12 @@ type Schedule struct { data map[string]map[string]Flight } -// ParseSchedule reads YAML from r into a Schedule. func ParseSchedule(r io.Reader) (*Schedule, error) { s := &Schedule{ data: make(map[string]map[string]Flight), } dec := yaml.NewDecoder(r) - // dec.KnownFields(true) // enable if you want strict field checking + // dec.KnownFields(true) // strict field checking if err := dec.Decode(&s.data); err != nil { return nil, err } diff --git a/pkg/localstore/import.go b/pkg/localstore/import.go deleted file mode 100644 index eb008ba..0000000 --- a/pkg/localstore/import.go +++ /dev/null @@ -1,484 +0,0 @@ -package localstore - -import ( - "encoding/csv" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "airlines/pkg/model" - - "github.com/schollz/progressbar/v3" -) - -func (s *Store) ImportAllCSVs(dir string) error { - if dir == "" { - return errors.New("empty directory path") - } - if !strings.HasSuffix(dir, string(filepath.Separator)) { - dir += string(filepath.Separator) - } - - // lock for writes while rebuilding everything - s.mu.Lock() - defer s.mu.Unlock() - - // reset containers - s.users = nil - s.cards = nil - s.flights = nil - if s.userFlights == nil { - s.userFlights = make(map[uint64]map[uint64]struct{}) - } else { - for k := range s.userFlights { - delete(s.userFlights, k) - } - } - if s.cardFlights == nil { - s.cardFlights = make(map[uint64]map[uint64]struct{}) - } else { - for k := range s.cardFlights { - delete(s.cardFlights, k) - } - } - // (Re)build helper indices when possible - if s.cardsByUser == nil { - s.cardsByUser = make(map[uint64]map[uint64]struct{}) - } else { - for k := range s.cardsByUser { - delete(s.cardsByUser, k) - } - } - // We cannot reconstruct codesByUser / countriesByUser from CSVs here; leave as-is or empty. - // Initialize if nil so your code using them won't panic. - if s.codesByUser == nil { - s.codesByUser = make(map[uint64]map[string]struct{}) - } - if s.countriesByUser == nil { - s.countriesByUser = make(map[uint64]map[string]struct{}) - } - - // 1) users.csv - if err := s.loadUsersCSV(dir + "users.csv"); err != nil { - return fmt.Errorf("load users.csv: %w", err) - } - fmt.Println("loaed users") - - // 2) cards.csv - if err := s.loadCardsCSV(dir + "cards.csv"); err != nil { - return fmt.Errorf("load cards.csv: %w", err) - } - - // 3) flights.csv - if err := s.loadFlightsCSV(dir + "flights.csv"); err != nil { - return fmt.Errorf("load flights.csv: %w", err) - } - - // 4) user_flights.csv - if err := s.loadUserFlightsCSV(dir + "user_flights.csv"); err != nil { - return fmt.Errorf("load user_flights.csv: %w", err) - } - - // 5) card_flights.csv - if err := s.loadCardFlightsCSV(dir + "card_flights.csv"); err != nil { - return fmt.Errorf("load card_flights.csv: %w", err) - } - - return nil -} - -func (s *Store) loadUsersCSV(path string) error { - r, closer, err := openCSV(path) - if err != nil { - return err - } - defer closer.Close() - - // header - if _, err := r.Read(); err != nil { - return fmt.Errorf("users header: %w", err) - } - - bar := progressbar.Default(int64(150000), "reading users") - - for { - bar.Add(1) - rec, err := r.Read() - if errors.Is(err, io.EOF) { - break - } - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("users read: %w", err) - } - // columns: id, nick, name, surname, fathersname, sex, birthday, total_flights, total_codes, total_countries, total_cards - if len(rec) < 11 { - return fmt.Errorf("users row has %d columns, expected >=11", len(rec)) - } - - id, err := parseUint(rec[0]) - if err != nil { - return fmt.Errorf("user id: %w", err) - } - sexInt, err := parseInt(rec[5]) - if err != nil { - return fmt.Errorf("user sex: %w", err) - } - - var bday time.Time - if strings.TrimSpace(rec[6]) != "" { - // "2006-01-02" in UTC - t, err := time.Parse("2006-01-02", rec[6]) - if err != nil { - return fmt.Errorf("user birthday: %w", err) - } - bday = t.UTC() - } else { - // keep zero time; or: - // bday = model.SentinelBirthday() // <-- if you prefer sentinel - } - - u := &model.User{ - ID: id, - Nick: rec[1], - Name: rec[2], - Surname: rec[3], - Fathersname: strings.TrimSpace(rec[4]), - Sex: model.Sex(sexInt), // adjust if your type differs - Birthday: bday, - } - - s.putUser(u) - } - return nil -} - -func (s *Store) loadCardsCSV(path string) error { - r, closer, err := openCSV(path) - if err != nil { - return err - } - defer closer.Close() - - // header - if _, err := r.Read(); err != nil { - return fmt.Errorf("cards header: %w", err) - } - - bar := progressbar.Default(int64(177000), "reading cards") - for { - bar.Add(1) - rec, err := r.Read() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("cards read: %w", err) - } - // columns: id, prefix, number, bonusprogramm, user_id - if len(rec) < 5 { - return fmt.Errorf("cards row has %d columns, expected >=5", len(rec)) - } - - id, err := parseUint(rec[0]) - if err != nil { - return fmt.Errorf("card id: %w", err) - } - num, err := parseUint(rec[2]) - if err != nil { - return fmt.Errorf("card number: %w", err) - } - uid, err := parseUint(rec[4]) - if err != nil { - return fmt.Errorf("card user_id: %w", err) - } - - c := &model.Card{ - ID: id, - Prefix: rec[1], - Number: num, - Bonusprogramm: rec[3], - UserID: uid, - } - s.putCard(c) - - // index: cardsByUser - if _, ok := s.cardsByUser[uid]; !ok { - s.cardsByUser[uid] = make(map[uint64]struct{}) - } - s.cardsByUser[uid][id] = struct{}{} - } - return nil -} - -func (s *Store) loadFlightsCSV(path string) error { - r, closer, err := openCSV(path) - if err != nil { - return err - } - defer closer.Close() - - // header - if _, err := r.Read(); err != nil { - return fmt.Errorf("flights header: %w", err) - } - - bar := progressbar.Default(int64(2000000), "reading flights") - for { - bar.Add(1) - rec, err := r.Read() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("flights read: %w", err) - } - // columns: - // id, number, from, to, fromlat, fromlon, tolat, tolon, dep_date, has_time, dep_time, dep_iso - if len(rec) < 12 { - return fmt.Errorf("flights row has %d columns, expected >=12", len(rec)) - } - - id, err := parseUint(rec[0]) - if err != nil { - return fmt.Errorf("flight id: %w", err) - } - - fromLat, err := parseFloat(rec[4]) - if err != nil { - return fmt.Errorf("fromlat: %w", err) - } - fromLon, err := parseFloat(rec[5]) - if err != nil { - return fmt.Errorf("fromlon: %w", err) - } - toLat, err := parseFloat(rec[6]) - if err != nil { - return fmt.Errorf("tolat: %w", err) - } - toLon, err := parseFloat(rec[7]) - if err != nil { - return fmt.Errorf("tolon: %w", err) - } - - depDateStr := strings.TrimSpace(rec[8]) // "2006-01-02" - hasTime, err := strconv.ParseBool(rec[9]) - if err != nil { - return fmt.Errorf("has_time: %w", err) - } - - var dep time.Time - if hasTime { - // When exported with time present, dep_iso is RFC3339; prefer it for full fidelity. - depISO := strings.TrimSpace(rec[11]) - if depISO != "" { - t, err := time.Parse(time.RFC3339, depISO) - if err != nil { - return fmt.Errorf("dep_iso: %w", err) - } - dep = t - } else { - // Fallback: combine dep_date + dep_time in local (treat as UTC if not specified) - depTime := strings.TrimSpace(rec[10]) // "15:04:05" - t, err := time.Parse("2006-01-02 15:04:05", depDateStr+" "+depTime) - if err != nil { - return fmt.Errorf("dep_date+dep_time: %w", err) - } - dep = t.UTC() - } - } else { - // Date only → set at UTC midnight of that date - if depDateStr == "" { - return fmt.Errorf("dep_date is empty while has_time=false") - } - t, err := time.Parse("2006-01-02", depDateStr) - if err != nil { - return fmt.Errorf("dep_date: %w", err) - } - dep = t.UTC() - } - - f := &model.Flight{ - ID: id, - Number: rec[1], - From: rec[2], - To: rec[3], - FromCoords: model.LatLong{ - Lat: fromLat, - Long: fromLon, - }, - ToCoords: model.LatLong{ - Lat: toLat, - Long: toLon, - }, - Date: dep, - HasTime: hasTime, - } - - s.putFlight(f) - } - return nil -} - -func (s *Store) loadUserFlightsCSV(path string) error { - r, closer, err := openCSV(path) - if err != nil { - return err - } - defer closer.Close() - - // header - if _, err := r.Read(); err != nil { - return fmt.Errorf("user_flights header: %w", err) - } - - bar := progressbar.Default(int64(3200000), "reading u-flights") - for { - bar.Add(1) - rec, err := r.Read() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("user_flights read: %w", err) - } - // columns: user_id, flight_id - if len(rec) < 2 { - return fmt.Errorf("user_flights row has %d columns, expected >=2", len(rec)) - } - uid, err := parseUint(rec[0]) - if err != nil { - return fmt.Errorf("user_id: %w", err) - } - fid, err := parseUint(rec[1]) - if err != nil { - return fmt.Errorf("flight_id: %w", err) - } - - // guard against missing references (mirror your exporter’s checks) - if !s.validFlightID(fid) { - continue - } - if _, ok := s.userFlights[uid]; !ok { - s.userFlights[uid] = make(map[uint64]struct{}) - } - s.userFlights[uid][fid] = struct{}{} - } - return nil -} - -func (s *Store) loadCardFlightsCSV(path string) error { - r, closer, err := openCSV(path) - if err != nil { - return err - } - defer closer.Close() - - // header - if _, err := r.Read(); err != nil { - return fmt.Errorf("card_flights header: %w", err) - } - - for { - rec, err := r.Read() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("card_flights read: %w", err) - } - // columns: card_id, flight_id - if len(rec) < 2 { - return fmt.Errorf("card_flights row has %d columns, expected >=2", len(rec)) - } - cid, err := parseUint(rec[0]) - if err != nil { - return fmt.Errorf("card_id: %w", err) - } - fid, err := parseUint(rec[1]) - if err != nil { - return fmt.Errorf("flight_id: %w", err) - } - - if !s.validFlightID(fid) { - continue - } - if _, ok := s.cardFlights[cid]; !ok { - s.cardFlights[cid] = make(map[uint64]struct{}) - } - s.cardFlights[cid][fid] = struct{}{} - } - return nil -} - -// --- helpers --- - -func openCSV(path string) (*csv.Reader, io.Closer, error) { - f, err := os.Open(path) - if err != nil { - return nil, nil, err - } - r := csv.NewReader(f) - r.ReuseRecord = true - // r.FieldsPerRecord = -1 // allow variable columns per row (comment out if you want strictness) - return r, f, nil -} - -func parseUint(s string) (uint64, error) { - return strconv.ParseUint(strings.TrimSpace(s), 10, 64) -} -func parseInt(s string) (int64, error) { - return strconv.ParseInt(strings.TrimSpace(s), 10, 64) -} -func parseFloat(s string) (float64, error) { - if strings.TrimSpace(s) == "" { - return 0, nil - } - return strconv.ParseFloat(strings.TrimSpace(s), 64) -} - -// Ensure slices are large enough and place item by ID (1-based supported) -func ensureLen[T any](slice []*T, id uint64) []*T { - needed := int(id) + 1 // keep index == id; index 0 unused per your exporters - if len(slice) <= needed { - newSlice := make([]*T, needed) - copy(newSlice, slice) - return newSlice - } - return slice -} - -func (s *Store) putUser(u *model.User) { - if u == nil { - return - } - s.users = ensureLen[model.User](s.users, u.ID) - s.users[u.ID] = u -} - -func (s *Store) putCard(c *model.Card) { - if c == nil { - return - } - s.cards = ensureLen[model.Card](s.cards, c.ID) - s.cards[c.ID] = c -} - -func (s *Store) putFlight(f *model.Flight) { - if f == nil { - return - } - s.flights = ensureLen[model.Flight](s.flights, f.ID) - s.flights[f.ID] = f -} - -func (s *Store) validFlightID(fid uint64) bool { - return fid != 0 && int(fid) < len(s.flights) && s.flights[fid] != nil -} diff --git a/pkg/localstore/store.go b/pkg/localstore/store.go index 151b2b4..dad2ffe 100644 --- a/pkg/localstore/store.go +++ b/pkg/localstore/store.go @@ -7,6 +7,7 @@ import ( "time" "unicode/utf8" + "airlines/pkg/airports" "airlines/pkg/model" ) @@ -51,7 +52,6 @@ type flightDayKey struct { DateYMD int32 } - type Store struct { mu sync.RWMutex @@ -97,7 +97,6 @@ func NewLocalStore() *Store { } } - func isZeroCoord(c model.LatLong) bool { return c.Lat == 0 && c.Long == 0 } func ymdUTC(t time.Time) int32 { @@ -122,8 +121,6 @@ func ensureSet(m map[uint64]map[uint64]struct{}, k uint64) map[uint64]struct{} { return s } -/* ============================== users ============================= */ - func fatherInitial(s string) string { s = strings.TrimSpace(s) if s == "" { @@ -133,7 +130,7 @@ func fatherInitial(s string) string { if r == utf8.RuneError { return "" } - return string(r) // your pipeline keeps it UPPER + return string(r) } func addUserInitIndex(m map[userInitKey]uint64, u *model.User) { @@ -148,7 +145,6 @@ func delUserInitIndex(m map[userInitKey]uint64, u *model.User) { delete(m, k) } -// --- merge helper (unchanged; keeps initial→full, birthday, nick, sex upgrades) --- func (s *Store) mergeUserFields(id uint64, in *model.User) *model.User { ex := s.users[id] // fathersname: initial -> full (same initial), move indexes @@ -186,12 +182,12 @@ func (s *Store) mergeUserFields(id uint64, in *model.User) *model.User { return ex } -// --- FIXED SaveUser: initial-key lookup tries BOTH (birth, 0) --- func (s *Store) SaveUser(u *model.User) (*model.User, error) { if u == nil { return nil, errors.New("nil user") } - // normalize (names already UPPER) + + // normalize just for sure u.Nick = strings.TrimSpace(u.Nick) u.Name = strings.TrimSpace(u.Name) u.Surname = strings.TrimSpace(u.Surname) @@ -205,14 +201,14 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { s.mu.Lock() defer s.mu.Unlock() - // 1) by Nick + // by Nick if u.Nick != "" { if id, ok := s.nickToUID[u.Nick]; ok { return s.mergeUserFields(id, u), nil } } - // 2) exact tuple + // exact tuple if id, ok := s.nameToUID[inKey]; ok { if u.Nick != "" && s.users[id].Nick == "" { s.users[id].Nick = u.Nick @@ -221,7 +217,7 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { return s.mergeUserFields(id, u), nil } - // 3) initial-based match (try with incoming birth, then with 0) + // try with incoming birth, then with 0 init := fatherInitial(u.Fathersname) tryInits := []userInitKey{ {Surname: u.Surname, Name: u.Name, Init: init, BirthYMD: inBirth}, @@ -233,14 +229,14 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { if id, ok := s.nameInitToUID[ik]; ok { ex := s.users[id] - // If ex has initial-only and incoming has full (same initial) → upgrade fathers + move indexes + // If ex has initial-only and incoming has full → upgrade fathers + move indexes if ex.Fathersname == fatherInitial(ex.Fathersname) && u.Fathersname != "" && fatherInitial(u.Fathersname) == fatherInitial(ex.Fathersname) { // move name index oldNameKey := userNameKey{Surname: ex.Surname, Name: ex.Name, Fathersname: ex.Fathersname, BirthYMD: ymdUTC(ex.Birthday)} delete(s.nameToUID, oldNameKey) - // remove old init index (birth may differ) + // remove old init index delUserInitIndex(s.nameInitToUID, ex) ex.Fathersname = u.Fathersname @@ -276,7 +272,7 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { } } - // 4) relaxed: fathersname empty, same birth + // fathersname empty, same birth if id, ok := s.nameToUID[userNameKey{Surname: u.Surname, Name: u.Name, Fathersname: "", BirthYMD: inBirth}]; ok { ex := s.users[id] if ex.Fathersname == "" && u.Fathersname != "" { @@ -298,7 +294,7 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { return ex, nil } - // 5) same fathersname, no birth + // same fathersname, no birth if id, ok := s.nameToUID[userNameKey{Surname: u.Surname, Name: u.Name, Fathersname: u.Fathersname, BirthYMD: 0}]; ok { delete(s.nameToUID, userNameKey{Surname: u.Surname, Name: u.Name, Fathersname: u.Fathersname, BirthYMD: 0}) ex := s.users[id] @@ -316,7 +312,7 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { return ex, nil } - // 6) fully unspecific existing (fathers="", birth=0) + // fathers="", birth=0 if id, ok := s.nameToUID[userNameKey{Surname: u.Surname, Name: u.Name, Fathersname: "", BirthYMD: 0}]; ok { ex := s.users[id] delete(s.nameToUID, userNameKey{Surname: ex.Surname, Name: ex.Name, Fathersname: "", BirthYMD: 0}) @@ -338,7 +334,7 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { return ex, nil } - // 7) create + // not found -> create u.ID = uint64(len(s.users)) s.users = append(s.users, u) if u.Nick != "" { @@ -349,15 +345,6 @@ func (s *Store) SaveUser(u *model.User) (*model.User, error) { return u, nil } -/* ============================== cards ============================= */ -/* -Match order: - 1) exact (Prefix, Number, Bonus) - 2) pair (Prefix, Number) → if stored bonus=="" and incoming bonus!="", upgrade in place (move triple index) - 3) else create new -Never steal UserID: only set if existing has 0 and incoming non-zero. -*/ - func (s *Store) SaveCard(c *model.Card) (*model.Card, error) { if c == nil { return nil, errors.New("nil card") @@ -404,7 +391,7 @@ func (s *Store) SaveCard(c *model.Card) (*model.Card, error) { s.cardsByUser[ex.UserID] = v switch { case ex.Bonusprogramm == "" && c.Bonusprogramm != "": - // move triple index from empty -> new bonus + // move index from empty -> new bonus oldTri := cardKey{Prefix: ex.Prefix, Number: ex.Number, Bonus: ex.Bonusprogramm} delete(s.cardToCID, oldTri) ex.Bonusprogramm = c.Bonusprogramm @@ -415,8 +402,8 @@ func (s *Store) SaveCard(c *model.Card) (*model.Card, error) { return ex, nil case ex.Bonusprogramm != "" && c.Bonusprogramm == "": return ex, nil - case ex.Bonusprogramm != "" && c.Bonusprogramm != "" && ex.Bonusprogramm != c.Bonusprogramm: - // different program → create new card record + // different program → create new card record + // case ex.Bonusprogramm != "" && c.Bonusprogramm != "" && ex.Bonusprogramm != c.Bonusprogramm: default: return ex, nil } @@ -426,7 +413,7 @@ func (s *Store) SaveCard(c *model.Card) (*model.Card, error) { c.ID = uint64(len(s.cards)) s.cards = append(s.cards, c) s.cardPairToCID[pair] = c.ID - s.cardToCID[tri] = c.ID // even if bonus == "", we still index triple + s.cardToCID[tri] = c.ID if s.cardsByUser[c.UserID] == nil { s.cardsByUser[c.UserID] = make(map[uint64]struct{}, 1024) @@ -438,18 +425,6 @@ func (s *Store) SaveCard(c *model.Card) (*model.Card, error) { return c, nil } -/* ============================== flights =========================== */ -/* -Identity: - - date-only: (Number, From, To, DateYMD, false, 0) - - timed : (Number, From, To, DateYMD, true, SecSinceMidnight) -Upgrade: - - if a date-only exists and a timed arrives for the same day, upgrade in place -Merge: - - coords: fill when missing - - relations: add (dedup via sets) -*/ - func (s *Store) SaveFlight(f *model.Flight) (*model.Flight, error) { if f == nil { return nil, errors.New("nil flight") @@ -474,14 +449,14 @@ func (s *Store) SaveFlight(f *model.Flight) (*model.Flight, error) { s.mu.Lock() defer s.mu.Unlock() - // 1) exact (precise) key + // precise key if id, ok := s.flightToFID[pKey]; ok { ex := s.flights[id] s.mergeFlightFields(id, ex, f) return ex, nil } - // 2) same day exists -> maybe upgrade date-only to timed + // same day exists -> maybe upgrade date-only to timed if id, ok := s.flightByDay[dayKey]; ok { ex := s.flights[id] exKey := s.keyOfFlight(ex) @@ -489,7 +464,7 @@ func (s *Store) SaveFlight(f *model.Flight) (*model.Flight, error) { // move map key to timed delete(s.flightToFID, exKey) ex.HasTime = true - // set clock from incoming (keep same calendar date) + // set clock from incoming ex.Date = time.Date(dayUTC.Year(), dayUTC.Month(), dayUTC.Day(), f.Date.Hour(), f.Date.Minute(), f.Date.Second(), f.Date.Nanosecond(), f.Date.Location()) s.flightToFID[s.keyOfFlight(ex)] = id @@ -500,20 +475,20 @@ func (s *Store) SaveFlight(f *model.Flight) (*model.Flight, error) { return ex, nil } - // 3) brand new + // brand new f.ID = uint64(len(s.flights)) s.flights = append(s.flights, f) s.flightToFID[pKey] = f.ID s.flightByDay[dayKey] = f.ID - // if s.countriesByUser[f.UserID] == nil { - // s.countriesByUser[f.UserID] = make(map[string]struct{}, 1024) - // } + if s.countriesByUser[f.UserID] == nil { + s.countriesByUser[f.UserID] = make(map[string]struct{}, 1024) + } - // v := s.countriesByUser[f.UserID] - // dd, _ := airports.LookupIATA(f.From) - // v[dd.Country] = struct{}{} - // s.countriesByUser[f.UserID] = v + v := s.countriesByUser[f.UserID] + dd, _ := airports.LookupIATA(f.From) + v[dd.Country] = struct{}{} + s.countriesByUser[f.UserID] = v if f.Code != "" { if s.codesByUser[f.UserID] == nil { @@ -543,7 +518,7 @@ func (s *Store) keyOfFlight(f *model.Flight) flightKey { } func (s *Store) mergeFlightFields(id uint64, ex, in *model.Flight) { - // coords: fill when empty + // coords fill when empty if isZeroCoord(ex.FromCoords) && !isZeroCoord(in.FromCoords) { ex.FromCoords = in.FromCoords } @@ -560,17 +535,15 @@ func (s *Store) mergeFlightFields(id uint64, ex, in *model.Flight) { if in.Code != "" && ex.Code == "" { ex.Code = in.Code - // if s.codesByUser[in.UserID] == nil { - // s.codesByUser[in.UserID] = make(map[string]struct{}, 1024) - // } - // codesByUser := s.codesByUser[in.UserID] - // codesByUser[in.Code] = struct{}{} - // s.codesByUser[in.UserID] = codesByUser + if s.codesByUser[in.UserID] == nil { + s.codesByUser[in.UserID] = make(map[string]struct{}, 1024) + } + codesByUser := s.codesByUser[in.UserID] + codesByUser[in.Code] = struct{}{} + s.codesByUser[in.UserID] = codesByUser } } -/* ============================== finders =========================== */ - func (s *Store) FindUserByNick(nick string) (*model.User, bool) { s.mu.RLock() defer s.mu.RUnlock() @@ -579,60 +552,4 @@ func (s *Store) FindUserByNick(nick string) (*model.User, bool) { return nil, false } return s.users[id], true -} - -func (s *Store) FindUserByName(name, surname, fathers string, bday time.Time) (*model.User, bool) { - key := userNameKey{ - Surname: strings.TrimSpace(surname), - Name: strings.TrimSpace(name), - Fathersname: strings.TrimSpace(fathers), - BirthYMD: ymdUTC(bday), - } - s.mu.RLock() - defer s.mu.RUnlock() - id, ok := s.nameToUID[key] - if !ok || id == 0 || int(id) >= len(s.users) { - return nil, false - } - return s.users[id], true -} - -func (s *Store) FindCard(prefix string, number uint64, bonus string) (*model.Card, bool) { - tri := cardKey{Prefix: strings.TrimSpace(prefix), Number: number, Bonus: strings.TrimSpace(bonus)} - s.mu.RLock() - defer s.mu.RUnlock() - if id, ok := s.cardToCID[tri]; ok && id != 0 && int(id) < len(s.cards) { - return s.cards[id], true - } - // fall back to pair if no exact - pair := cardPairKey{Prefix: strings.TrimSpace(prefix), Number: number} - if id, ok := s.cardPairToCID[pair]; ok && id != 0 && int(id) < len(s.cards) { - return s.cards[id], true - } - return nil, false -} - -func (s *Store) FindFlight(number, from, to string, date time.Time, hasTime bool) (*model.Flight, bool) { - number = strings.TrimSpace(number) - from = strings.TrimSpace(from) - to = strings.TrimSpace(to) - - ymd := ymdUTC(time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC)) - var k flightKey - if hasTime { - k = flightKey{Number: number, From: from, To: to, DateYMD: ymd, HasTime: true, Sec: secSinceMidnight(date)} - } else { - k = flightKey{Number: number, From: from, To: to, DateYMD: ymd, HasTime: false, Sec: 0} - } - - s.mu.RLock() - defer s.mu.RUnlock() - if id, ok := s.flightToFID[k]; ok && id != 0 && int(id) < len(s.flights) { - return s.flights[id], true - } - // day-level fallback (returns best precision for the day if exact key absent) - if id, ok := s.flightByDay[flightDayKey{Number: number, From: from, To: to, DateYMD: ymd}]; ok && id != 0 && int(id) < len(s.flights) { - return s.flights[id], true - } - return nil, false -} +} \ No newline at end of file diff --git a/pkg/model/card.go b/pkg/model/card.go index 6ed666b..5c9de76 100644 --- a/pkg/model/card.go +++ b/pkg/model/card.go @@ -11,18 +11,15 @@ func ParseCardLine(s string) (prefix string, number uint64, bonus string) { if raw == "" { return "", 0, "" } - // number = last run of digits if m := regexp.MustCompile(`(\d{3,})\D*$`).FindStringSubmatch(raw); len(m) == 2 { if n, err := strconv.ParseUint(m[1], 10, 64); err == nil { number = n } } - // tokens (letters with '-', '/', apostrophes) tokRe := regexp.MustCompile(`[A-Za-z][A-Za-z'/-]*`) toks := tokRe.FindAllString(s, -1) - // prefix = first 2–3 letter all-caps-ish token for _, t := range toks { u := strings.ToUpper(t) if len(u) >= 2 && len(u) <= 3 && regexp.MustCompile(`^[A-Z]{2,3}$`).MatchString(u) { @@ -30,7 +27,7 @@ func ParseCardLine(s string) (prefix string, number uint64, bonus string) { break } } - // bonus = all tokens except prefix + words := []string{} for _, t := range toks { if strings.ToUpper(t) == prefix { diff --git a/pkg/model/model.go b/pkg/model/model.go deleted file mode 100644 index 0a7e026..0000000 --- a/pkg/model/model.go +++ /dev/null @@ -1,7 +0,0 @@ -package model - - - -type Adapter interface { - Init(string) // inits adapter with filename -} \ No newline at end of file diff --git a/pkg/model/types.go b/pkg/model/types.go index fd65d46..8b1106d 100644 --- a/pkg/model/types.go +++ b/pkg/model/types.go @@ -33,7 +33,7 @@ func (s *Sex) UnmarshalJSON(b []byte) error { return nil } } - // also accept numbers in JSON + var n int if err := json.Unmarshal(b, &n); err == nil { *s = Sex(n) diff --git a/pkg/names/fio.go b/pkg/names/fio.go index 4ecca7e..0b83633 100644 --- a/pkg/names/fio.go +++ b/pkg/names/fio.go @@ -13,8 +13,6 @@ type Parts struct { Patronymic string // may be "" or an initial like "F" } -// ParseLatinName parses 2–3 tokens containing First/Last and optional patronymic (1–2 letters). -// Tokens may be in any order, e.g. "PETROVSKAYA KARINA" or "RUSLAN F EVSEEV". func ParseLatinName(s string) (Parts, error) { toks := tokenizeLatin(s) // keeps letters, apostrophes, hyphens, optional trailing dot if len(toks) < 2 || len(toks) > 3 { @@ -31,16 +29,16 @@ func ParseLatinName(s string) (Parts, error) { ps = append(ps, part{raw: t, lo: lo}) } - // 1) Patronymic: 1–2 letters (optionally with a trailing dot), or RU-style patronymic suffix + // fathersname pIdx := -1 for i, p := range ps { - if isInitial(p.raw) || isPatronymicLatin(p.lo) { + if isInitial(p.raw) || isFathersnameLatin(p.lo) { pIdx = i break } } - // 2) Surname: look for common last-name suffixes among remaining tokens + // Surname lIdx := -1 for i, p := range ps { if i == pIdx { @@ -52,7 +50,7 @@ func ParseLatinName(s string) (Parts, error) { } } - // 3) Assign the rest to first name; tie-break if needed + // firs name rem := make([]int, 0, 2) for i := range ps { if i != pIdx && i != lIdx { @@ -60,7 +58,7 @@ func ParseLatinName(s string) (Parts, error) { } } - // If surname not obvious and we have 2 leftovers, pick the longer one as surname + // if surname not obvious and we have 2 leftovers, pick the longer one as surname ;) if lIdx == -1 && len(rem) == 2 { if runeLen(ps[rem[0]].raw) >= runeLen(ps[rem[1]].raw) { lIdx = rem[0] @@ -79,7 +77,7 @@ func ParseLatinName(s string) (Parts, error) { out.Last = ps[lIdx].raw } - // Remaining becomes first name; if still empty (2 tokens), pick the non-surname/non-patronymic as first + // remaining becomes first name if len(rem) == 1 { out.First = ps[rem[0]].raw } else if len(ps) == 2 { @@ -90,12 +88,12 @@ func ParseLatinName(s string) (Parts, error) { } } - // Normalize to Title Case (capitalize first letter, lowercase rest) + // normalize to Title Case out.First = capWord(out.First) out.Last = capWord(out.Last) out.Patronymic = strings.ToUpper(out.Patronymic) // keep initials uppercase - // Sanity + // not found ;( if out.First == "" || out.Last == "" { return out, errors.New("unable to classify parts") } @@ -103,7 +101,6 @@ func ParseLatinName(s string) (Parts, error) { } func tokenizeLatin(s string) []string { - // keep letters, apostrophes, hyphens; allow an optional trailing dot for initials re := regexp.MustCompile(`(?i)[a-z]+(?:['-][a-z]+)*\.?`) return re.FindAllString(s, -1) } @@ -114,8 +111,7 @@ func isInitial(x string) bool { return len(r) >= 1 && len(r) <= 2 && allASCIIAlpha(r) } -func isPatronymicLatin(lo string) bool { - // Latin transliterations of RU patronymics (very rough) +func isFathersnameLatin(lo string) bool { sufs := []string{"ovich", "evich", "ich", "ovna", "evna", "ichna", "ogly", "kyzy"} for _, s := range sufs { if strings.HasSuffix(lo, s) && len(lo) >= len(s)+2 { @@ -126,7 +122,6 @@ func isPatronymicLatin(lo string) bool { } func looksLikeSurnameLatin(lo string) bool { - // Common Slavic surname endings (male & female forms) sufs := []string{ "ov", "ev", "in", "ina", "ova", "eva", "sky", "skiy", "skii", "skaya", "ska", @@ -140,9 +135,7 @@ func looksLikeSurnameLatin(lo string) bool { return true } } - // If token contains an apostrophe mid-word (e.g., emel'yanova), still may be a surname if strings.Contains(lo, "'") { - // feminine -'yanova/-'eva etc. if strings.HasSuffix(lo, "yanova") || strings.HasSuffix(lo, "yanov") || strings.HasSuffix(lo, "eva") || strings.HasSuffix(lo, "ova") { return true } @@ -154,7 +147,6 @@ func capWord(s string) string { if s == "" { return s } - // keep internal hyphens/apostrophes, title-case each segment sep := func(r rune) bool { return r == '-' || r == '\'' } parts := strings.FieldsFunc(strings.ToLower(s), sep) i := 0 @@ -164,7 +156,7 @@ func capWord(s string) string { builder.WriteRune(r) continue } - // find which sub-part this rune belongs to by counting letters consumed + if len(parts) == 0 { builder.WriteRune(unicode.ToUpper(r)) continue @@ -175,10 +167,8 @@ func capWord(s string) string { builder.WriteRune(unicode.ToLower(r)) } i++ - // crude reset at separators handled above } - // Simpler/robust alternative: - // return strings.Title(strings.ToLower(s)) // deprecated but OK for ASCII; avoided here. + return strings.ToUpper(string([]rune(s)[0])) + strings.ToLower(s[1:]) } diff --git a/pkg/names/gender.go b/pkg/names/gender.go index bdd16e1..4282d9d 100644 --- a/pkg/names/gender.go +++ b/pkg/names/gender.go @@ -8,7 +8,7 @@ import ( func normalizeTitle(x string) string { x = strings.ToLower(x) - // strip common punctuation + // strip punctuation x = strings.ReplaceAll(x, ".", "") x = strings.ReplaceAll(x, "'", "") x = strings.ReplaceAll(x, "’", "") @@ -20,7 +20,6 @@ func GenderFromTitle(s string) model.Sex { if s == "" { return model.SexUnknown } - // only first token (before space/comma/slash/etc.) cut := func(r rune) bool { return unicode.IsSpace(r) || r == ',' || r == '/' || r == '&' } first := strings.FieldsFunc(s, cut) if len(first) == 0 { @@ -28,20 +27,19 @@ func GenderFromTitle(s string) model.Sex { } t := normalizeTitle(first[0]) - // male honorifics + // male switch t { - case "mr", "sir", "lord", "monsieur", "m", "don", "senor", "sr": // "sr" may collide with "senior"; context needed + case "mr", "sir", "lord", "monsieur", "m", "don", "senor", "sr": return model.SexMale } - // female honorifics + // female switch t { case "mrs", "miss", "ms", "madam", "madame", "mademoiselle", "mlle", "lady", "dame", "senora", "sra", "señora", "srta", "srita", "dona": return model.SexFemale } - // neutral/ambiguous titles (return Unknown) - // e.g., "mx", "dr", "prof", "rev", "coach", "officer", etc. + // flying helicopter return model.SexUnknown } diff --git a/pkg/store/db.go b/pkg/store/db.go index 6853563..38bdbaa 100644 --- a/pkg/store/db.go +++ b/pkg/store/db.go @@ -20,8 +20,8 @@ type Store struct { func NewStore(dsn string) (*Store, error) { db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ - SkipDefaultTransaction: true, // you can wrap outside for big imports - PrepareStmt: true, // statement cache + SkipDefaultTransaction: true, + PrepareStmt: true, DisableNestedTransaction: true, Logger: logger.Default.LogMode(logger.Silent), }) @@ -35,8 +35,6 @@ func (s *Store) AutoMigrate() error { return s.DB.AutoMigrate(&model.User{}, &model.Card{}, &model.Flight{}) } -/* ============================= Users ============================= */ - func (s *Store) SaveUser(ctx context.Context, in *model.User) (*model.User, error) { if in == nil { return nil, errors.New("nil user") @@ -52,7 +50,7 @@ func (s *Store) SaveUser(ctx context.Context, in *model.User) (*model.User, erro out := &model.User{} err := s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - // 1) Try by Nick + // try by Nick if in.Nick != "" { if err := tx.Where("nick = ?", in.Nick).First(out).Error; err == nil { return nil @@ -60,7 +58,7 @@ func (s *Store) SaveUser(ctx context.Context, in *model.User) (*model.User, erro return err } } - // 2) Fallback by identity tuple + // fallback by identity tuple q := tx.Where("name = ? AND surname = ?", in.Name, in.Surname) if in.Fathersname != "" { q = q.Where("fathersname = ?", in.Fathersname) @@ -76,7 +74,7 @@ func (s *Store) SaveUser(ctx context.Context, in *model.User) (*model.User, erro return err } - // 3) Not found → create + // create if err := tx.Create(in).Error; err != nil { return err } @@ -86,9 +84,6 @@ func (s *Store) SaveUser(ctx context.Context, in *model.User) (*model.User, erro return out, err } -/* ============================= Cards ============================= */ -/* Card→User is a plain FK (cards.user_id). Keep it. */ - func (s *Store) SaveCard(ctx context.Context, in *model.Card) (*model.Card, error) { if in == nil { return nil, errors.New("nil card") @@ -104,14 +99,14 @@ func (s *Store) SaveCard(ctx context.Context, in *model.Card) (*model.Card, erro // find by unique triple if err := tx.Where("prefix = ? AND number = ? AND bonusprogramm = ?", in.Prefix, in.Number, in.Bonusprogramm).First(out).Error; err == nil { - // Link to user once if requested and not yet linked + // link to user once if requested and not yet linked if in.UserID != 0 && out.UserID == 0 { if err := tx.Model(out).Update("user_id", in.UserID).Error; err != nil { return err } out.UserID = in.UserID } - // refuse stealing if different user already set + // refuse stealing if in.UserID != 0 && out.UserID != 0 && out.UserID != in.UserID { return errors.New("card already linked to another user") } @@ -120,7 +115,7 @@ func (s *Store) SaveCard(ctx context.Context, in *model.Card) (*model.Card, erro return err } - // not found → create (includes FK if provided) + // not found → create if err := tx.Create(in).Error; err != nil { return err } @@ -130,15 +125,6 @@ func (s *Store) SaveCard(ctx context.Context, in *model.Card) (*model.Card, erro return out, err } -/* ============================= Flights ============================= */ -/* - Find/create by (number, from, to, date). Do NOT write user_id/card_id columns. - Use associations to link: - - flight.Users ↔ user_flights - - flight.Cards ↔ card_flights - This avoids “raw IDs” on the flight row and uses the relations instead. -*/ - func (s *Store) SaveFlight(ctx context.Context, in *model.Flight) (*model.Flight, error) { if in == nil { return nil, errors.New("nil flight") @@ -147,12 +133,12 @@ func (s *Store) SaveFlight(ctx context.Context, in *model.Flight) (*model.Flight out := &model.Flight{} err := s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - // Try to find existing + // try to find existing if err := tx.Where(`"number" = ? AND "from" = ? AND "to" = ? AND "date" = ?`, in.Number, in.From, in.To, in.Date).First(out).Error; err == nil { - // found: do NOT update columns; only link relations below + // found } else if errors.Is(err, gorm.ErrRecordNotFound) { - // Not found → create (no user_id/card_id columns are written) + // not found → create cre := model.Flight{ Number: in.Number, From: in.From, @@ -162,7 +148,7 @@ func (s *Store) SaveFlight(ctx context.Context, in *model.Flight) (*model.Flight if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&cre).Error; err != nil { return err } - // If conflict, fetch the existing one + // on conflict, fetch the existing one if cre.ID == 0 { if err := tx.Where(`"number" = ? AND "from" = ? AND "to" = ? AND "date" = ?`, in.Number, in.From, in.To, in.Date).First(&cre).Error; err != nil { @@ -174,16 +160,14 @@ func (s *Store) SaveFlight(ctx context.Context, in *model.Flight) (*model.Flight return err } - // ---- Link relations via associations (no raw IDs in the row) ---- - // Link to User (many-to-many) if caller provided a UserID + // many-to-many user-flight if in.UserID != 0 { u := model.User{ID: in.UserID} - // Avoid dup join rows by adding a unique index on (user_id, flight_id). if err := tx.Model(out).Association("Users").Append(&u); err != nil { return err } } - // Link to Card (many-to-many) + // many-to-many flight-card if in.CardID != 0 { c := model.Card{ID: in.CardID} if err := tx.Model(out).Association("Cards").Append(&c); err != nil { -- cgit v1.2.3