Skip to content

ScoutingSystemV2 / SSEData

Go to a project
Commit e1242bc3 by Augusto
Options

advanced importers

parent 32efaacb
Showing with 236 additions and 147 deletions
docs/swagger.yaml
...@@ -905,29 +905,47 @@ paths: ...@@ -905,29 +905,47 @@ paths:
summary: Import competitions from TheSports summary: Import competitions from TheSports
tags: tags:
- Import - Import
/import/matches/diary: /import/matches/advancedstats:
post: post:
description: Performs a matches import using TheSports match/diary API for a description: 'Single: provide matchWyId (imports both teams'' match advanced
given date or 24h window. The `date` query parameter (YYYY-MM-DD) is recommended; stats). Auto: omit matchWyId (imports all matches with wy_id in matches table;
if omitted, the provider default will be used (usually current day). resumable).'
parameters: parameters:
- description: Date in YYYY-MM-DD format for which to import the schedule/results - description: Wyscout match ID (optional; omit for auto mode)
in: query in: query
name: date name: matchWyId
type: string type: integer
- description: 'Optional limit on number of requests (auto: matches)'
in: query
name: limit
type: integer
- description: 'Auto mode only: concurrent workers (default 4)'
in: query
name: workers
type: integer
- description: 'Auto mode only: reset checkpoint and restart from beginning'
in: query
name: reset
type: boolean
responses: responses:
"200": "200":
description: OK description: OK
schema: schema:
additionalProperties: true additionalProperties: true
type: object type: object
"400":
description: Bad Request
schema:
additionalProperties:
type: string
type: object
"500": "500":
description: Internal Server Error description: Internal Server Error
schema: schema:
additionalProperties: additionalProperties:
type: string type: string
type: object type: object
summary: Import matches diary from TheSports summary: Import match advanced stats from Wyscout
tags: tags:
- Import - Import
/import/matches/fixtures: /import/matches/fixtures:
...@@ -999,107 +1017,6 @@ paths: ...@@ -999,107 +1017,6 @@ paths:
summary: Import match formations from Wyscout v4 summary: Import match formations from Wyscout v4
tags: tags:
- Import - Import
/import/matches/lineup:
post:
description: Performs a lineup import using TheSports match/lineup/detail API.
If 'matchTsId' is provided, imports lineup for a specific match. If omitted,
processes matches from the last 30 days only (API limitation). Use 'limit'
for testing and 'batchSize' to control memory usage.
parameters:
- description: TheSports match id (tsId) for which to import the lineup (optional;
if omitted, processes matches from last 30 days)
in: query
name: matchTsId
type: string
- description: 'Maximum number of matches to process in batch mode (default:
no limit; useful for debugging)'
in: query
name: limit
type: integer
- description: 'Number of matches to load per batch (default: 1000; lower for
memory constraints)'
in: query
name: batchSize
type: integer
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import match lineups from TheSports
tags:
- Import
/import/matches/list:
post:
description: Performs a full import of all matches from TheSports match/list
API using pagination. This is intended for one-time initial sync to get all
historical matches. The API returns 1000 matches per page and stops when total
is 0. Use startPage to resume from a specific page if the import was interrupted.
parameters:
- description: 'Starting page number (default: 1, use to resume interrupted
import)'
in: query
name: startPage
type: integer
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import all matches from TheSports (one-time full sync)
tags:
- Import
/import/matches/recent:
post:
description: Performs a matches import using TheSports match/recent/list API.
If `since` is provided (unix seconds), only matches updated since that time
are fetched using the time-based endpoint. Otherwise, a full import is performed
using page-based pagination (last 30 days).
parameters:
- description: Page size per request (default 100, only used for full imports)
in: query
name: pageSize
type: integer
- description: Unix timestamp (seconds) to import only matches updated since
this time
in: query
name: since
type: integer
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import recent matches from TheSports
tags:
- Import
/import/matches/wyscout: /import/matches/wyscout:
post: post:
description: Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/matches. description: Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/matches.
...@@ -1465,6 +1382,57 @@ paths: ...@@ -1465,6 +1382,57 @@ paths:
summary: Import teams from TheSports summary: Import teams from TheSports
tags: tags:
- Import - Import
/import/teams/advancedstats:
post:
description: 'Single: provide teamWyId+competitionId+seasonId. Auto: omit all
IDs (imports distinct team+competition+season combos derived from matches;
resumable).'
parameters:
- description: Wyscout team ID (optional; omit for auto mode)
in: query
name: teamWyId
type: integer
- description: Wyscout competition ID (required for single; omit for auto mode)
in: query
name: competitionId
type: integer
- description: Wyscout season ID (required for single; omit for auto mode)
in: query
name: seasonId
type: integer
- description: 'Optional limit on number of requests (auto: combos)'
in: query
name: limit
type: integer
- description: 'Auto mode only: concurrent workers (default 4)'
in: query
name: workers
type: integer
- description: 'Auto mode only: reset checkpoint and restart from beginning'
in: query
name: reset
type: boolean
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import team advanced stats from Wyscout
tags:
- Import
/import/teams/career: /import/teams/career:
post: post:
description: Fetches /v3/teams/{teamWyId}/career?details=competition,season description: Fetches /v3/teams/{teamWyId}/career?details=competition,season
......
internal/database/database.go
...@@ -2,6 +2,7 @@ package database ...@@ -2,6 +2,7 @@ package database
import ( import (
"fmt" "fmt"
"os"
"gorm.io/driver/postgres" "gorm.io/driver/postgres"
"gorm.io/gorm" "gorm.io/gorm"
...@@ -25,33 +26,35 @@ func Connect(cfg config.Config) (*gorm.DB, error) { ...@@ -25,33 +26,35 @@ func Connect(cfg config.Config) (*gorm.DB, error) {
return nil, err return nil, err
} }
if err := db.AutoMigrate( if os.Getenv("DB_AUTOMIGRATE") == "true" {
&models.Area{}, if err := db.AutoMigrate(
&models.Competition{}, &models.Area{},
&models.Season{}, &models.Competition{},
&models.Round{}, &models.Season{},
&models.Team{}, &models.Round{},
&models.TeamChild{}, &models.Team{},
&models.TeamCareer{}, &models.TeamChild{},
&models.TeamAdvancedStats{}, &models.TeamCareer{},
&models.Coach{}, &models.TeamAdvancedStats{},
&models.Referee{}, &models.Coach{},
&models.Player{}, &models.Referee{},
&models.Match{}, &models.Player{},
&models.MatchAdvancedStats{}, &models.Match{},
&models.MatchTeam{}, &models.MatchAdvancedStats{},
&models.MatchLineupPlayer{}, &models.MatchTeam{},
&models.MatchFormation{}, &models.MatchLineupPlayer{},
&models.PlayerTransfer{}, &models.MatchFormation{},
&models.PlayerCareer{}, &models.PlayerTransfer{},
&models.PlayerAdvancedStats{}, &models.PlayerCareer{},
&models.PlayerAdvancedPosition{}, &models.PlayerAdvancedStats{},
&models.TeamSquad{}, &models.PlayerAdvancedPosition{},
&models.Standing{}, &models.TeamSquad{},
&models.ImportCheckpoint{}, &models.Standing{},
&models.SampleRecord{}, &models.ImportCheckpoint{},
); err != nil { &models.SampleRecord{},
return nil, err ); err != nil {
return nil, err
}
} }
return db, nil return db, nil
......
internal/handlers/players.go
...@@ -263,6 +263,108 @@ func (h *PlayerHandler) GetAdvancedStatsAverages(c *gin.Context) { ...@@ -263,6 +263,108 @@ func (h *PlayerHandler) GetAdvancedStatsAverages(c *gin.Context) {
} }
} }
var allRows []models.PlayerAdvancedStats
if err := h.DB.WithContext(c.Request.Context()).
Where("player_wy_id = ?", wyID).
Order("season_id DESC").
Find(&allRows).Error; err != nil {
respondError(c, err)
return
}
type competitionSummary struct {
WyID int `json:"wyId"`
Name *string `json:"name,omitempty"`
Logo *string `json:"logo,omitempty"`
}
type seasonSummary struct {
WyID int `json:"wyId"`
Name *string `json:"name,omitempty"`
}
type advancedStatsOut struct {
models.PlayerAdvancedStats
Season *seasonSummary `json:"season,omitempty"`
Competition *competitionSummary `json:"competition,omitempty"`
}
seasonIDs := make([]int, 0, len(allRows))
competitionIDs := make([]int, 0, len(allRows))
seenSeason := map[int]struct{}{}
seenCompetition := map[int]struct{}{}
for _, row := range allRows {
if row.SeasonID > 0 {
if _, ok := seenSeason[row.SeasonID]; !ok {
seenSeason[row.SeasonID] = struct{}{}
seasonIDs = append(seasonIDs, row.SeasonID)
}
}
if row.CompetitionID > 0 {
if _, ok := seenCompetition[row.CompetitionID]; !ok {
seenCompetition[row.CompetitionID] = struct{}{}
competitionIDs = append(competitionIDs, row.CompetitionID)
}
}
}
seasonsByWyID := map[int]seasonSummary{}
if len(seasonIDs) > 0 {
var seasons []struct {
WyID int `gorm:"column:wy_id"`
Name string `gorm:"column:name"`
}
if err := h.DB.WithContext(c.Request.Context()).
Model(&models.Season{}).
Select("wy_id", "name").
Where("wy_id IN ?", seasonIDs).
Find(&seasons).Error; err == nil {
for _, s := range seasons {
name := s.Name
seasonsByWyID[s.WyID] = seasonSummary{WyID: s.WyID, Name: &name}
}
}
}
competitionsByWyID := map[int]competitionSummary{}
if len(competitionIDs) > 0 {
var comps []struct {
WyID int `gorm:"column:wy_id"`
Name string `gorm:"column:name"`
Logo *string `gorm:"column:logo"`
}
if err := h.DB.WithContext(c.Request.Context()).
Model(&models.Competition{}).
Select("wy_id", "name", "logo").
Where("wy_id IN ?", competitionIDs).
Find(&comps).Error; err == nil {
for _, comp := range comps {
name := comp.Name
var logo *string
if comp.Logo != nil && *comp.Logo != "" {
v := *comp.Logo
logo = &v
}
competitionsByWyID[comp.WyID] = competitionSummary{WyID: comp.WyID, Name: &name, Logo: logo}
}
}
}
outItems := make([]advancedStatsOut, 0, len(allRows))
for _, row := range allRows {
var season *seasonSummary
if v, ok := seasonsByWyID[row.SeasonID]; ok {
tmp := v
season = &tmp
}
var competition *competitionSummary
if v, ok := competitionsByWyID[row.CompetitionID]; ok {
tmp := v
competition = &tmp
}
outItems = append(outItems, advancedStatsOut{PlayerAdvancedStats: row, Season: season, Competition: competition})
}
timestamp := time.Now().UTC().Format(time.RFC3339) timestamp := time.Now().UTC().Format(time.RFC3339)
endpoint := fmt.Sprintf("/players/wyscout/%d/advancedstats", wyID) endpoint := fmt.Sprintf("/players/wyscout/%d/advancedstats", wyID)
...@@ -274,6 +376,7 @@ func (h *PlayerHandler) GetAdvancedStatsAverages(c *gin.Context) { ...@@ -274,6 +376,7 @@ func (h *PlayerHandler) GetAdvancedStatsAverages(c *gin.Context) {
"wyId": anchorSeasonID, "wyId": anchorSeasonID,
"name": anchorSeasonName, "name": anchorSeasonName,
}, },
"items": outItems,
"seasonAvg": aggregateNumericPointers(seasonRows), "seasonAvg": aggregateNumericPointers(seasonRows),
"last2YearsAvg": aggregateNumericPointers(last2Rows), "last2YearsAvg": aggregateNumericPointers(last2Rows),
"last5YearsAvg": aggregateNumericPointers(last5Rows), "last5YearsAvg": aggregateNumericPointers(last5Rows),
...@@ -1199,6 +1302,7 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) { ...@@ -1199,6 +1302,7 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) {
WyID int `json:"wyId"` WyID int `json:"wyId"`
Name *string `json:"name,omitempty"` Name *string `json:"name,omitempty"`
Image *string `json:"image,omitempty"` Image *string `json:"image,omitempty"`
Type *string `json:"type,omitempty"`
} }
type competitionSummary struct { type competitionSummary struct {
...@@ -1253,10 +1357,11 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) { ...@@ -1253,10 +1357,11 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) {
WyID int `gorm:"column:wy_id"` WyID int `gorm:"column:wy_id"`
Name string `gorm:"column:name"` Name string `gorm:"column:name"`
ImageDataURL *string `gorm:"column:image_data_url"` ImageDataURL *string `gorm:"column:image_data_url"`
Type string `gorm:"column:type"`
} }
if err := h.DB.WithContext(c.Request.Context()). if err := h.DB.WithContext(c.Request.Context()).
Model(&models.Team{}). Model(&models.Team{}).
Select("wy_id", "name", "image_data_url"). Select("wy_id", "name", "image_data_url", "type").
Where("wy_id IN ?", teamIDs). Where("wy_id IN ?", teamIDs).
Find(&teams).Error; err == nil { Find(&teams).Error; err == nil {
for _, t := range teams { for _, t := range teams {
...@@ -1266,7 +1371,12 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) { ...@@ -1266,7 +1371,12 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) {
v := *t.ImageDataURL v := *t.ImageDataURL
img = &v img = &v
} }
teamsByWyID[t.WyID] = teamSummary{WyID: t.WyID, Name: &name, Image: img} var typ *string
if t.Type != "" {
v := t.Type
typ = &v
}
teamsByWyID[t.WyID] = teamSummary{WyID: t.WyID, Name: &name, Image: img, Type: typ}
} }
} }
} }
......
migrations/0017_add_player_name_search_indexes.sql
...@@ -7,6 +7,14 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm; ...@@ -7,6 +7,14 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm;
-- Enable unaccent extension for accent-insensitive search (if not already enabled) -- Enable unaccent extension for accent-insensitive search (if not already enabled)
CREATE EXTENSION IF NOT EXISTS unaccent; CREATE EXTENSION IF NOT EXISTS unaccent;
CREATE OR REPLACE FUNCTION unaccent_immutable(text)
RETURNS text
LANGUAGE sql
IMMUTABLE
AS $$
SELECT unaccent($1);
$$;
-- Add GIN trigram indexes for fast ILIKE searches on name columns -- Add GIN trigram indexes for fast ILIKE searches on name columns
-- These indexes dramatically speed up pattern matching queries like ILIKE '%search%' -- These indexes dramatically speed up pattern matching queries like ILIKE '%search%'
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_short_name_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_short_name_trgm
...@@ -60,28 +68,28 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_short_name_trgm ...@@ -60,28 +68,28 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_short_name_trgm
-- Accent-insensitive trigram indexes (support searches where user input may omit accents) -- Accent-insensitive trigram indexes (support searches where user input may omit accents)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_short_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_short_name_unaccent_trgm
ON players USING GIN (unaccent(LOWER(short_name)) gin_trgm_ops); ON players USING GIN (unaccent_immutable(LOWER(short_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_first_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_first_name_unaccent_trgm
ON players USING GIN (unaccent(LOWER(first_name)) gin_trgm_ops); ON players USING GIN (unaccent_immutable(LOWER(first_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_last_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_last_name_unaccent_trgm
ON players USING GIN (unaccent(LOWER(last_name)) gin_trgm_ops); ON players USING GIN (unaccent_immutable(LOWER(last_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_teams_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_teams_name_unaccent_trgm
ON teams USING GIN (unaccent(LOWER(name)) gin_trgm_ops); ON teams USING GIN (unaccent_immutable(LOWER(name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_teams_short_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_teams_short_name_unaccent_trgm
ON teams USING GIN (unaccent(LOWER(short_name)) gin_trgm_ops); ON teams USING GIN (unaccent_immutable(LOWER(short_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_first_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_first_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(first_name)) gin_trgm_ops); ON coaches USING GIN (unaccent_immutable(LOWER(first_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_middle_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_middle_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(middle_name)) gin_trgm_ops); ON coaches USING GIN (unaccent_immutable(LOWER(middle_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_last_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_last_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(last_name)) gin_trgm_ops); ON coaches USING GIN (unaccent_immutable(LOWER(last_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_short_name_unaccent_trgm CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_short_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(short_name)) gin_trgm_ops); ON coaches USING GIN (unaccent_immutable(LOWER(short_name)) gin_trgm_ops);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment