Skip to content

ScoutingSystemV2 / SSEData

Go to a project
Commit e1242bc3 by Augusto
Options

advanced importers

parent 32efaacb
Showing with 435 additions and 1462 deletions
docs/docs.go
......@@ -750,59 +750,36 @@ const docTemplate = `{
}
}
},
"/import/matches/diary": {
"/import/matches/advancedstats": {
"post": {
"description": "Performs a matches import using TheSports match/diary API for a given date or 24h window. The ` + "`" + `date` + "`" + ` query parameter (YYYY-MM-DD) is recommended; if omitted, the provider default will be used (usually current day).",
"description": "Single: provide matchWyId (imports both teams' match advanced stats). Auto: omit matchWyId (imports all matches with wy_id in matches table; resumable).",
"tags": [
"Import"
],
"summary": "Import matches diary from TheSports",
"summary": "Import match advanced stats from Wyscout",
"parameters": [
{
"type": "string",
"description": "Date in YYYY-MM-DD format for which to import the schedule/results",
"name": "date",
"type": "integer",
"description": "Wyscout match ID (optional; omit for auto mode)",
"name": "matchWyId",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/matches/fixtures": {
"post": {
"description": "Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/fixtures?details=matches. If seasonWyId is omitted, imports for all seasons in the DB with a wy_id.",
"tags": [
"Import"
],
"summary": "Import matches from Wyscout fixtures",
"parameters": [
{
"type": "integer",
"description": "Wyscout season ID",
"name": "seasonWyId",
"description": "Optional limit on number of requests (auto: matches)",
"name": "limit",
"in": "query"
},
{
"type": "integer",
"description": "Limit number of seasons processed when seasonWyId is omitted",
"name": "limitSeasons",
"description": "Auto mode only: concurrent workers (default 4)",
"name": "workers",
"in": "query"
},
{
"type": "boolean",
"description": "Auto mode only: reset checkpoint and restart from beginning",
"name": "reset",
"in": "query"
}
],
......@@ -835,24 +812,24 @@ const docTemplate = `{
}
}
},
"/import/matches/formations": {
"/import/matches/fixtures": {
"post": {
"description": "Fetches /v4/matches/{matchWyId}/formations for a given matchWyId. If matchWyId is omitted, processes matches in the DB with a wy_id. Stores formations in match_formations.",
"description": "Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/fixtures?details=matches. If seasonWyId is omitted, imports for all seasons in the DB with a wy_id.",
"tags": [
"Import"
],
"summary": "Import match formations from Wyscout v4",
"summary": "Import matches from Wyscout fixtures",
"parameters": [
{
"type": "integer",
"description": "Wyscout match wy_id",
"name": "matchWyId",
"description": "Wyscout season ID",
"name": "seasonWyId",
"in": "query"
},
{
"type": "integer",
"description": "Limit number of matches processed when matchWyId is omitted",
"name": "limit",
"description": "Limit number of seasons processed when seasonWyId is omitted",
"name": "limitSeasons",
"in": "query"
}
],
......@@ -885,30 +862,24 @@ const docTemplate = `{
}
}
},
"/import/matches/lineup": {
"/import/matches/formations": {
"post": {
"description": "Performs a lineup import using TheSports match/lineup/detail API. If 'matchTsId' is provided, imports lineup for a specific match. If omitted, processes matches from the last 30 days only (API limitation). Use 'limit' for testing and 'batchSize' to control memory usage.",
"description": "Fetches /v4/matches/{matchWyId}/formations for a given matchWyId. If matchWyId is omitted, processes matches in the DB with a wy_id. Stores formations in match_formations.",
"tags": [
"Import"
],
"summary": "Import match lineups from TheSports",
"summary": "Import match formations from Wyscout v4",
"parameters": [
{
"type": "string",
"description": "TheSports match id (tsId) for which to import the lineup (optional; if omitted, processes matches from last 30 days)",
"name": "matchTsId",
"in": "query"
},
{
"type": "integer",
"description": "Maximum number of matches to process in batch mode (default: no limit; useful for debugging)",
"name": "limit",
"description": "Wyscout match wy_id",
"name": "matchWyId",
"in": "query"
},
{
"type": "integer",
"description": "Number of matches to load per batch (default: 1000; lower for memory constraints)",
"name": "batchSize",
"description": "Limit number of matches processed when matchWyId is omitted",
"name": "limit",
"in": "query"
}
],
......@@ -941,82 +912,6 @@ const docTemplate = `{
}
}
},
"/import/matches/list": {
"post": {
"description": "Performs a full import of all matches from TheSports match/list API using pagination. This is intended for one-time initial sync to get all historical matches. The API returns 1000 matches per page and stops when total is 0. Use startPage to resume from a specific page if the import was interrupted.",
"tags": [
"Import"
],
"summary": "Import all matches from TheSports (one-time full sync)",
"parameters": [
{
"type": "integer",
"description": "Starting page number (default: 1, use to resume interrupted import)",
"name": "startPage",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/matches/recent": {
"post": {
"description": "Performs a matches import using TheSports match/recent/list API. If ` + "`" + `since` + "`" + ` is provided (unix seconds), only matches updated since that time are fetched using the time-based endpoint. Otherwise, a full import is performed using page-based pagination (last 30 days).",
"tags": [
"Import"
],
"summary": "Import recent matches from TheSports",
"parameters": [
{
"type": "integer",
"description": "Page size per request (default 100, only used for full imports)",
"name": "pageSize",
"in": "query"
},
{
"type": "integer",
"description": "Unix timestamp (seconds) to import only matches updated since this time",
"name": "since",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/matches/wyscout": {
"post": {
"description": "Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/matches. Requires seasonWyId query param.",
......@@ -1531,6 +1426,80 @@ const docTemplate = `{
}
}
},
"/import/teams/advancedstats": {
"post": {
"description": "Single: provide teamWyId+competitionId+seasonId. Auto: omit all IDs (imports distinct team+competition+season combos derived from matches; resumable).",
"tags": [
"Import"
],
"summary": "Import team advanced stats from Wyscout",
"parameters": [
{
"type": "integer",
"description": "Wyscout team ID (optional; omit for auto mode)",
"name": "teamWyId",
"in": "query"
},
{
"type": "integer",
"description": "Wyscout competition ID (required for single; omit for auto mode)",
"name": "competitionId",
"in": "query"
},
{
"type": "integer",
"description": "Wyscout season ID (required for single; omit for auto mode)",
"name": "seasonId",
"in": "query"
},
{
"type": "integer",
"description": "Optional limit on number of requests (auto: combos)",
"name": "limit",
"in": "query"
},
{
"type": "integer",
"description": "Auto mode only: concurrent workers (default 4)",
"name": "workers",
"in": "query"
},
{
"type": "boolean",
"description": "Auto mode only: reset checkpoint and restart from beginning",
"name": "reset",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"400": {
"description": "Bad Request",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/teams/career": {
"post": {
"description": "Fetches /v3/teams/{teamWyId}/career?details=competition,season for teams in the DB (teams.wy_id not null) plus teams referenced in team_children. Upserts records into team_careers.",
......
docs/swagger.json
......@@ -743,59 +743,36 @@
}
}
},
"/import/matches/diary": {
"/import/matches/advancedstats": {
"post": {
"description": "Performs a matches import using TheSports match/diary API for a given date or 24h window. The `date` query parameter (YYYY-MM-DD) is recommended; if omitted, the provider default will be used (usually current day).",
"description": "Single: provide matchWyId (imports both teams' match advanced stats). Auto: omit matchWyId (imports all matches with wy_id in matches table; resumable).",
"tags": [
"Import"
],
"summary": "Import matches diary from TheSports",
"summary": "Import match advanced stats from Wyscout",
"parameters": [
{
"type": "string",
"description": "Date in YYYY-MM-DD format for which to import the schedule/results",
"name": "date",
"type": "integer",
"description": "Wyscout match ID (optional; omit for auto mode)",
"name": "matchWyId",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/matches/fixtures": {
"post": {
"description": "Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/fixtures?details=matches. If seasonWyId is omitted, imports for all seasons in the DB with a wy_id.",
"tags": [
"Import"
],
"summary": "Import matches from Wyscout fixtures",
"parameters": [
{
"type": "integer",
"description": "Wyscout season ID",
"name": "seasonWyId",
"description": "Optional limit on number of requests (auto: matches)",
"name": "limit",
"in": "query"
},
{
"type": "integer",
"description": "Limit number of seasons processed when seasonWyId is omitted",
"name": "limitSeasons",
"description": "Auto mode only: concurrent workers (default 4)",
"name": "workers",
"in": "query"
},
{
"type": "boolean",
"description": "Auto mode only: reset checkpoint and restart from beginning",
"name": "reset",
"in": "query"
}
],
......@@ -828,24 +805,24 @@
}
}
},
"/import/matches/formations": {
"/import/matches/fixtures": {
"post": {
"description": "Fetches /v4/matches/{matchWyId}/formations for a given matchWyId. If matchWyId is omitted, processes matches in the DB with a wy_id. Stores formations in match_formations.",
"description": "Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/fixtures?details=matches. If seasonWyId is omitted, imports for all seasons in the DB with a wy_id.",
"tags": [
"Import"
],
"summary": "Import match formations from Wyscout v4",
"summary": "Import matches from Wyscout fixtures",
"parameters": [
{
"type": "integer",
"description": "Wyscout match wy_id",
"name": "matchWyId",
"description": "Wyscout season ID",
"name": "seasonWyId",
"in": "query"
},
{
"type": "integer",
"description": "Limit number of matches processed when matchWyId is omitted",
"name": "limit",
"description": "Limit number of seasons processed when seasonWyId is omitted",
"name": "limitSeasons",
"in": "query"
}
],
......@@ -878,30 +855,24 @@
}
}
},
"/import/matches/lineup": {
"/import/matches/formations": {
"post": {
"description": "Performs a lineup import using TheSports match/lineup/detail API. If 'matchTsId' is provided, imports lineup for a specific match. If omitted, processes matches from the last 30 days only (API limitation). Use 'limit' for testing and 'batchSize' to control memory usage.",
"description": "Fetches /v4/matches/{matchWyId}/formations for a given matchWyId. If matchWyId is omitted, processes matches in the DB with a wy_id. Stores formations in match_formations.",
"tags": [
"Import"
],
"summary": "Import match lineups from TheSports",
"summary": "Import match formations from Wyscout v4",
"parameters": [
{
"type": "string",
"description": "TheSports match id (tsId) for which to import the lineup (optional; if omitted, processes matches from last 30 days)",
"name": "matchTsId",
"in": "query"
},
{
"type": "integer",
"description": "Maximum number of matches to process in batch mode (default: no limit; useful for debugging)",
"name": "limit",
"description": "Wyscout match wy_id",
"name": "matchWyId",
"in": "query"
},
{
"type": "integer",
"description": "Number of matches to load per batch (default: 1000; lower for memory constraints)",
"name": "batchSize",
"description": "Limit number of matches processed when matchWyId is omitted",
"name": "limit",
"in": "query"
}
],
......@@ -934,82 +905,6 @@
}
}
},
"/import/matches/list": {
"post": {
"description": "Performs a full import of all matches from TheSports match/list API using pagination. This is intended for one-time initial sync to get all historical matches. The API returns 1000 matches per page and stops when total is 0. Use startPage to resume from a specific page if the import was interrupted.",
"tags": [
"Import"
],
"summary": "Import all matches from TheSports (one-time full sync)",
"parameters": [
{
"type": "integer",
"description": "Starting page number (default: 1, use to resume interrupted import)",
"name": "startPage",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/matches/recent": {
"post": {
"description": "Performs a matches import using TheSports match/recent/list API. If `since` is provided (unix seconds), only matches updated since that time are fetched using the time-based endpoint. Otherwise, a full import is performed using page-based pagination (last 30 days).",
"tags": [
"Import"
],
"summary": "Import recent matches from TheSports",
"parameters": [
{
"type": "integer",
"description": "Page size per request (default 100, only used for full imports)",
"name": "pageSize",
"in": "query"
},
{
"type": "integer",
"description": "Unix timestamp (seconds) to import only matches updated since this time",
"name": "since",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/matches/wyscout": {
"post": {
"description": "Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/matches. Requires seasonWyId query param.",
......@@ -1524,6 +1419,80 @@
}
}
},
"/import/teams/advancedstats": {
"post": {
"description": "Single: provide teamWyId+competitionId+seasonId. Auto: omit all IDs (imports distinct team+competition+season combos derived from matches; resumable).",
"tags": [
"Import"
],
"summary": "Import team advanced stats from Wyscout",
"parameters": [
{
"type": "integer",
"description": "Wyscout team ID (optional; omit for auto mode)",
"name": "teamWyId",
"in": "query"
},
{
"type": "integer",
"description": "Wyscout competition ID (required for single; omit for auto mode)",
"name": "competitionId",
"in": "query"
},
{
"type": "integer",
"description": "Wyscout season ID (required for single; omit for auto mode)",
"name": "seasonId",
"in": "query"
},
{
"type": "integer",
"description": "Optional limit on number of requests (auto: combos)",
"name": "limit",
"in": "query"
},
{
"type": "integer",
"description": "Auto mode only: concurrent workers (default 4)",
"name": "workers",
"in": "query"
},
{
"type": "boolean",
"description": "Auto mode only: reset checkpoint and restart from beginning",
"name": "reset",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"400": {
"description": "Bad Request",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/import/teams/career": {
"post": {
"description": "Fetches /v3/teams/{teamWyId}/career?details=competition,season for teams in the DB (teams.wy_id not null) plus teams referenced in team_children. Upserts records into team_careers.",
......
docs/swagger.yaml
......@@ -905,29 +905,47 @@ paths:
summary: Import competitions from TheSports
tags:
- Import
/import/matches/diary:
/import/matches/advancedstats:
post:
description: Performs a matches import using TheSports match/diary API for a
given date or 24h window. The `date` query parameter (YYYY-MM-DD) is recommended;
if omitted, the provider default will be used (usually current day).
description: 'Single: provide matchWyId (imports both teams'' match advanced
stats). Auto: omit matchWyId (imports all matches with wy_id in matches table;
resumable).'
parameters:
- description: Date in YYYY-MM-DD format for which to import the schedule/results
- description: Wyscout match ID (optional; omit for auto mode)
in: query
name: date
type: string
name: matchWyId
type: integer
- description: 'Optional limit on number of requests (auto: matches)'
in: query
name: limit
type: integer
- description: 'Auto mode only: concurrent workers (default 4)'
in: query
name: workers
type: integer
- description: 'Auto mode only: reset checkpoint and restart from beginning'
in: query
name: reset
type: boolean
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import matches diary from TheSports
summary: Import match advanced stats from Wyscout
tags:
- Import
/import/matches/fixtures:
......@@ -999,107 +1017,6 @@ paths:
summary: Import match formations from Wyscout v4
tags:
- Import
/import/matches/lineup:
post:
description: Performs a lineup import using TheSports match/lineup/detail API.
If 'matchTsId' is provided, imports lineup for a specific match. If omitted,
processes matches from the last 30 days only (API limitation). Use 'limit'
for testing and 'batchSize' to control memory usage.
parameters:
- description: TheSports match id (tsId) for which to import the lineup (optional;
if omitted, processes matches from last 30 days)
in: query
name: matchTsId
type: string
- description: 'Maximum number of matches to process in batch mode (default:
no limit; useful for debugging)'
in: query
name: limit
type: integer
- description: 'Number of matches to load per batch (default: 1000; lower for
memory constraints)'
in: query
name: batchSize
type: integer
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import match lineups from TheSports
tags:
- Import
/import/matches/list:
post:
description: Performs a full import of all matches from TheSports match/list
API using pagination. This is intended for one-time initial sync to get all
historical matches. The API returns 1000 matches per page and stops when total
is 0. Use startPage to resume from a specific page if the import was interrupted.
parameters:
- description: 'Starting page number (default: 1, use to resume interrupted
import)'
in: query
name: startPage
type: integer
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import all matches from TheSports (one-time full sync)
tags:
- Import
/import/matches/recent:
post:
description: Performs a matches import using TheSports match/recent/list API.
If `since` is provided (unix seconds), only matches updated since that time
are fetched using the time-based endpoint. Otherwise, a full import is performed
using page-based pagination (last 30 days).
parameters:
- description: Page size per request (default 100, only used for full imports)
in: query
name: pageSize
type: integer
- description: Unix timestamp (seconds) to import only matches updated since
this time
in: query
name: since
type: integer
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import recent matches from TheSports
tags:
- Import
/import/matches/wyscout:
post:
description: Imports matches for a given Wyscout season via /v3/seasons/{seasonWyId}/matches.
......@@ -1465,6 +1382,57 @@ paths:
summary: Import teams from TheSports
tags:
- Import
/import/teams/advancedstats:
post:
description: 'Single: provide teamWyId+competitionId+seasonId. Auto: omit all
IDs (imports distinct team+competition+season combos derived from matches;
resumable).'
parameters:
- description: Wyscout team ID (optional; omit for auto mode)
in: query
name: teamWyId
type: integer
- description: Wyscout competition ID (required for single; omit for auto mode)
in: query
name: competitionId
type: integer
- description: Wyscout season ID (required for single; omit for auto mode)
in: query
name: seasonId
type: integer
- description: 'Optional limit on number of requests (auto: combos)'
in: query
name: limit
type: integer
- description: 'Auto mode only: concurrent workers (default 4)'
in: query
name: workers
type: integer
- description: 'Auto mode only: reset checkpoint and restart from beginning'
in: query
name: reset
type: boolean
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal Server Error
schema:
additionalProperties:
type: string
type: object
summary: Import team advanced stats from Wyscout
tags:
- Import
/import/teams/career:
post:
description: Fetches /v3/teams/{teamWyId}/career?details=competition,season
......
internal/database/database.go
......@@ -2,6 +2,7 @@ package database
import (
"fmt"
"os"
"gorm.io/driver/postgres"
"gorm.io/gorm"
......@@ -25,6 +26,7 @@ func Connect(cfg config.Config) (*gorm.DB, error) {
return nil, err
}
if os.Getenv("DB_AUTOMIGRATE") == "true" {
if err := db.AutoMigrate(
&models.Area{},
&models.Competition{},
......@@ -53,6 +55,7 @@ func Connect(cfg config.Config) (*gorm.DB, error) {
); err != nil {
return nil, err
}
}
return db, nil
}
internal/handlers/import.go
......@@ -1390,612 +1390,6 @@ func (h *ImportHandler) ImportSeasons(c *gin.Context) {
})
}
// ImportMatchLineup imports match lineups from TheSports API (match/lineup/detail).
// @Summary Import match lineups from TheSports
// @Description Performs a lineup import using TheSports match/lineup/detail API. If 'matchTsId' is provided, imports lineup for a specific match. If omitted, processes matches from the last 30 days only (API limitation). Use 'limit' for testing and 'batchSize' to control memory usage.
// @Tags Import
// @Param matchTsId query string false "TheSports match id (tsId) for which to import the lineup (optional; if omitted, processes matches from last 30 days)"
// @Param limit query int false "Maximum number of matches to process in batch mode (default: no limit; useful for debugging)"
// @Param batchSize query int false "Number of matches to load per batch (default: 1000; lower for memory constraints)"
// @Success 200 {object} map[string]interface{}
// @Failure 400 {object} map[string]string
// @Failure 500 {object} map[string]string
// @Router /import/matches/lineup [post]
func (h *ImportHandler) ImportMatchLineup(c *gin.Context) {
user := h.Cfg.ProviderUser
secret := h.Cfg.ProviderSecret
if user == "" || secret == "" {
c.JSON(http.StatusInternalServerError, gin.H{"error": "ProviderUser/ProviderSecret not configured"})
return
}
matchTsIDParam := c.Query("matchTsId")
limitStr := c.DefaultQuery("limit", "")
batchSizeStr := c.DefaultQuery("batchSize", "1000")
var limit int
if limitStr != "" {
if v, err := strconv.Atoi(limitStr); err == nil && v > 0 {
limit = v
}
}
batchSize, err := strconv.Atoi(batchSizeStr)
if err != nil || batchSize <= 0 || batchSize > 10000 {
batchSize = 1000 // default and sanity check
}
// TheSports lineup detail: we focus on team and player-level fields we need; other fields are ignored.
type lineupPlayer struct {
ID string `json:"id"`
ShirtNumber int `json:"shirt_number"`
Position string `json:"position"`
X int `json:"x"`
Y int `json:"y"`
First int `json:"first"` // 1 for starter, 0 for sub
Captain int `json:"captain"` // 1 for captain, 0 for not
Rating string `json:"rating"`
Incidents []struct {
Type int `json:"type"`
Time string `json:"time"`
Minute int `json:"minute"`
AddTime int `json:"addtime"`
Belong int `json:"belong"`
HomeScore int `json:"home_score"`
AwayScore int `json:"away_score"`
} `json:"incidents"`
}
var payload struct {
Code int `json:"code"`
Results struct {
HomeFormation string `json:"home_formation"`
AwayFormation string `json:"away_formation"`
CoachID struct {
Home string `json:"home"`
Away string `json:"away"`
} `json:"coach_id"`
Lineup struct {
Home []lineupPlayer `json:"home"`
Away []lineupPlayer `json:"away"`
} `json:"lineup"`
} `json:"results"`
}
// helper to upsert team and players for one side
processSide := func(matchTsID string, side string, players []lineupPlayer, coachID, formation string, createdTeams *int, createdPlayers *int, errorsCount *int) {
if len(players) == 0 {
return
}
// First, get the team ID from the first player (we'll need to map this)
// For now, we'll need to find the team from the match
var match models.Match
if err := h.DB.Where("ts_id = ?", matchTsID).First(&match).Error; err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to get match for team mapping: %v\n", err)
(*errorsCount)++
return
}
// Determine team TsID based on side
var teamTsID *string
if side == "home" {
teamTsID = match.HomeTeamTsID
} else {
teamTsID = match.AwayTeamTsID
}
if teamTsID == nil || *teamTsID == "" {
fmt.Printf("ImportMatchLineup: ERROR - no team TsID found for %s side\n", side)
(*errorsCount)++
return
}
mt := models.MatchTeam{
MatchTsID: matchTsID,
Side: side,
}
mt.TeamTsID = teamTsID
if formation != "" {
mt.Formation = &formation
}
// resolve team WyID if available
var team models.Team
if err := h.DB.Where("ts_id = ?", *teamTsID).First(&team).Error; err == nil {
mt.TeamWyID = team.WyID
}
// resolve coach WyID if available
if coachID != "" {
var coach models.Coach
if err := h.DB.Where("ts_id = ?", coachID).First(&coach).Error; err == nil {
mt.CoachWyID = coach.WyID
mt.CoachTsID = &coachID
}
}
if err := h.DB.Create(&mt).Error; err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to create match team for %s: %v\n", side, err)
(*errorsCount)++
return
}
(*createdTeams)++
// Process players
subOrder := 1
for _, p := range players {
mlu := models.MatchLineupPlayer{
MatchTsID: matchTsID,
}
if mt.TeamWyID != nil {
mlu.TeamWyID = mt.TeamWyID
}
if mt.TeamTsID != nil {
mlu.TeamTsID = mt.TeamTsID
}
if p.ID != "" {
mlu.PlayerTsID = &p.ID
var pl models.Player
if err := h.DB.Where("ts_id = ?", p.ID).First(&pl).Error; err == nil {
mlu.PlayerWyID = pl.WyID
}
}
// Map shirt number
shirtNum := p.ShirtNumber
mlu.ShirtNumber = &shirtNum
// Map position
if p.Position != "" {
pos := p.Position
mlu.Position = &pos
}
// Map coordinates (convert int to float64)
posX := float64(p.X)
posY := float64(p.Y)
mlu.PosX = &posX
mlu.PosY = &posY
// Map starter status
mlu.IsStarter = (p.First == 1)
// Handle incidents for substitution times
if len(p.Incidents) > 0 {
for _, incident := range p.Incidents {
// Type 8 is substitution (based on typical sports API codes)
if incident.Type == 8 {
if incident.Minute > 0 {
mlu.MinuteIn = &incident.Minute
}
if incident.AddTime > 0 {
totalMinute := incident.Minute + incident.AddTime
mlu.MinuteOut = &totalMinute
}
}
}
}
so := subOrder
mlu.SubOrder = &so
subOrder++
if err := h.DB.Create(&mlu).Error; err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to create match lineup player %s: %v\n", p.ID, err)
(*errorsCount)++
continue
}
(*createdPlayers)++
}
}
// helper to import lineup for a single matchTsID
importForMatch := func(matchTsID string) (int, int, int) {
createdTeams := 0
createdPlayers := 0
errorsCount := 0
// ensure match exists
var match models.Match
if err := h.DB.Where("ts_id = ?", matchTsID).First(&match).Error; err != nil {
if err == gorm.ErrRecordNotFound {
fmt.Printf("ImportMatchLineup: ERROR - match not found for ts_id=%s\n", matchTsID)
return 0, 0, 1
}
fmt.Printf("ImportMatchLineup: ERROR - failed to retrieve match for ts_id=%s: %v\n", matchTsID, err)
return 0, 0, 1
}
url := fmt.Sprintf("https://api.thesports.com/v1/football/match/lineup/detail?user=%s&secret=%s&uuid=%s", user, secret, matchTsID)
resp, err := h.Client.Get(url)
if err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to call TheSports match lineup API for %s: %v\n", matchTsID, err)
return 0, 0, 1
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
fmt.Printf("ImportMatchLineup: SKIP - match not found in TheSports API for matchTsId=%s (404)\n", matchTsID)
return 0, 0, 0 // Don't count as error, just skip
}
fmt.Printf("ImportMatchLineup: provider returned status %d for matchTsId=%s\n", resp.StatusCode, matchTsID)
return 0, 0, 1
}
// Read response body
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to read response body for %s: %v\n", matchTsID, err)
return 0, 0, 1
}
resp.Body.Close()
if err := json.Unmarshal(bodyBytes, &payload); err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to decode provider response for matchTsId=%s: %v\n", matchTsID, err)
return 0, 0, 1
}
if payload.Code != 0 {
fmt.Printf("ImportMatchLineup: provider returned code=%d for matchTsId=%s\n", payload.Code, matchTsID)
return 0, 0, 1
}
// clear existing lineup for this match
if err := h.DB.Where("match_ts_id = ?", matchTsID).Delete(&models.MatchLineupPlayer{}).Error; err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to clear existing lineup data for matchTsId=%s: %v\n", matchTsID, err)
return 0, 0, 1
}
if err := h.DB.Where("match_ts_id = ?", matchTsID).Delete(&models.MatchTeam{}).Error; err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to clear existing match teams for matchTsId=%s: %v\n", matchTsID, err)
return 0, 0, 1
}
// Process home and away lineups with the correct structure
processSide(matchTsID, "home", payload.Results.Lineup.Home,
payload.Results.CoachID.Home, payload.Results.HomeFormation,
&createdTeams, &createdPlayers, &errorsCount)
processSide(matchTsID, "away", payload.Results.Lineup.Away,
payload.Results.CoachID.Away, payload.Results.AwayFormation,
&createdTeams, &createdPlayers, &errorsCount)
fmt.Printf("ImportMatchLineup: SUCCESS - imported lineup for matchTsId=%s (teams: %d, players: %d)\n", matchTsID, createdTeams, createdPlayers)
return createdTeams, createdPlayers, errorsCount
}
totalTeams := 0
totalPlayers := 0
totalErrors := 0
matchesProcessed := 0
matchesFailed := 0
if matchTsIDParam != "" {
// single-match mode
ct, cp, ce := importForMatch(matchTsIDParam)
matchesProcessed++
if ce > 0 {
matchesFailed++
}
totalTeams += ct
totalPlayers += cp
totalErrors += ce
} else {
// batch mode: process matches from last 30 days only (API limitation)
thirtyDaysAgo := time.Now().UTC().AddDate(0, 0, -30)
fmt.Printf("ImportMatchLineup: starting batch import for matches since %s (last 30 days) with pageSize=%d\n",
thirtyDaysAgo.Format("2006-01-02"), batchSize)
offset := 0
processedCount := 0
for {
var matches []models.Match
query := h.DB.Where("ts_id <> '' AND match_date >= ?", thirtyDaysAgo).
Offset(offset).Limit(batchSize)
if limit > 0 && processedCount+batchSize > limit {
remaining := limit - processedCount
if remaining <= 0 {
break
}
query = query.Limit(remaining)
}
if err := query.Find(&matches).Error; err != nil {
fmt.Printf("ImportMatchLineup: ERROR - failed to fetch batch at offset %d: %v\n", offset, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list matches for lineup import"})
return
}
if len(matches) == 0 {
break // no more matches
}
fmt.Printf("ImportMatchLineup: processing batch %d-%d (%d matches)\n",
offset+1, offset+len(matches), len(matches))
for _, m := range matches {
if m.TsID == nil || *m.TsID == "" {
continue
}
ct, cp, ce := importForMatch(*m.TsID)
matchesProcessed++
if ce > 0 {
matchesFailed++
}
totalTeams += ct
totalPlayers += cp
totalErrors += ce
// Progress indicator
if matchesProcessed%100 == 0 {
fmt.Printf("ImportMatchLineup: progress - %d matches processed\n", matchesProcessed)
}
}
processedCount += len(matches)
offset += batchSize
// Break if we've hit the limit
if limit > 0 && processedCount >= limit {
break
}
// If we got fewer than batchSize, we're done
if len(matches) < batchSize {
break
}
}
fmt.Printf("ImportMatchLineup: batch import completed - processed %d matches from last 30 days\n", matchesProcessed)
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Match lineup import completed",
"data": gin.H{
"teams": totalTeams,
"players": totalPlayers,
"errors": totalErrors,
"matchesProcessed": matchesProcessed,
"matchesFailed": matchesFailed,
},
"timestamp": time.Now().UTC().Format(time.RFC3339),
})
}
// ImportMatchesDiary imports matches for a given day from TheSports API (match/diary).
// @Summary Import matches diary from TheSports
// @Description Performs a matches import using TheSports match/diary API for a given date or 24h window. The `date` query parameter (YYYY-MM-DD) is recommended; if omitted, the provider default will be used (usually current day).
// @Tags Import
// @Param date query string false "Date in YYYY-MM-DD format for which to import the schedule/results"
// @Success 200 {object} map[string]interface{}
// @Failure 500 {object} map[string]string
// @Router /import/matches/diary [post]
func (h *ImportHandler) ImportMatchesDiary(c *gin.Context) {
user := h.Cfg.ProviderUser
secret := h.Cfg.ProviderSecret
if user == "" || secret == "" {
c.JSON(http.StatusInternalServerError, gin.H{"error": "ProviderUser/ProviderSecret not configured"})
return
}
date := c.Query("date")
baseURL := "https://api.thesports.com/v1/football/match/diary"
url := fmt.Sprintf("%s?user=%s&secret=%s&type=diary", baseURL, user, secret)
if date != "" {
url = fmt.Sprintf("%s&date=%s", url, date)
}
imported, updated, errorsCount := 0, 0, 0
resp, err := h.Client.Get(url)
if err != nil {
log.Printf("failed to call TheSports matches diary API: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to call provider API"})
return
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("provider returned status %d", resp.StatusCode)})
return
}
var payload struct {
Query struct {
Total int `json:"total"`
} `json:"query"`
Results []struct {
ID string `json:"id"`
SeasonID string `json:"season_id"`
CompetitionID string `json:"competition_id"`
HomeTeamID string `json:"home_team_id"`
AwayTeamID string `json:"away_team_id"`
StatusID int `json:"status_id"`
MatchTime int64 `json:"match_time"`
VenueID string `json:"venue_id"`
RefereeID string `json:"referee_id"`
Neutral int `json:"neutral"`
Note string `json:"note"`
HomeScores []int `json:"home_scores"`
AwayScores []int `json:"away_scores"`
HomePosition string `json:"home_position"`
AwayPosition string `json:"away_position"`
Coverage struct {
MLive int `json:"mlive"`
Lineup int `json:"lineup"`
} `json:"coverage"`
Round struct {
StageID string `json:"stage_id"`
GroupNum int `json:"group_num"`
RoundNum int `json:"round_num"`
} `json:"round"`
RelatedID string `json:"related_id"`
AggScore []int `json:"agg_score"`
Environment struct {
Weather int `json:"weather"`
Pressure string `json:"pressure"`
Temperature string `json:"temperature"`
Wind string `json:"wind"`
Humidity string `json:"humidity"`
} `json:"environment"`
TBD int `json:"tbd"`
HasOT int `json:"has_ot"`
Ended int `json:"ended"`
TeamReverse int `json:"team_reverse"`
Loss int `json:"loss"`
UpdatedAt int64 `json:"updated_at"`
} `json:"results"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
resp.Body.Close()
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to decode provider response"})
return
}
resp.Body.Close()
if payload.Query.Total == 0 || len(payload.Results) == 0 {
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Matches diary import completed (no results)",
"data": gin.H{
"imported": 0,
"updated": 0,
"errors": 0,
},
"timestamp": time.Now().UTC().Format(time.RFC3339),
})
return
}
for _, r := range payload.Results {
var match models.Match
if err := h.DB.Where("ts_id = ?", r.ID).First(&match).Error; err != nil {
if err == gorm.ErrRecordNotFound {
tsID := r.ID
match = models.Match{
TsID: &tsID,
}
if r.MatchTime > 0 {
mt := time.Unix(r.MatchTime, 0).UTC()
match.MatchDate = mt
}
if r.HomeTeamID != "" {
match.HomeTeamTsID = &r.HomeTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.HomeTeamID).First(&team).Error; err == nil {
match.HomeTeamWyID = team.WyID
}
}
if r.AwayTeamID != "" {
match.AwayTeamTsID = &r.AwayTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.AwayTeamID).First(&team).Error; err == nil {
match.AwayTeamWyID = team.WyID
}
}
if len(r.HomeScores) > 0 {
match.HomeScore = r.HomeScores[0]
}
if len(r.AwayScores) > 0 {
match.AwayScore = r.AwayScores[0]
}
if len(r.HomeScores) > 6 {
match.HomeScorePenalties = r.HomeScores[6]
}
if len(r.AwayScores) > 6 {
match.AwayScorePenalties = r.AwayScores[6]
}
if r.Environment.Temperature != "" {
if temp, err := strconv.ParseFloat(r.Environment.Temperature, 64); err == nil {
match.Temperature = &temp
}
}
if r.Environment.Weather != 0 {
w := fmt.Sprintf("%d", r.Environment.Weather)
match.Weather = &w
}
if r.UpdatedAt > 0 {
ut := time.Unix(r.UpdatedAt, 0).UTC()
match.APILastSyncedAt = &ut
match.APISyncStatus = "synced"
}
if err := h.DB.Create(&match).Error; err != nil {
errorsCount++
continue
}
imported++
continue
}
errorsCount++
continue
}
// update existing match
if r.MatchTime > 0 {
mt := time.Unix(r.MatchTime, 0).UTC()
match.MatchDate = mt
}
if r.HomeTeamID != "" {
match.HomeTeamTsID = &r.HomeTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.HomeTeamID).First(&team).Error; err == nil {
match.HomeTeamWyID = team.WyID
}
}
if r.AwayTeamID != "" {
match.AwayTeamTsID = &r.AwayTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.AwayTeamID).First(&team).Error; err == nil {
match.AwayTeamWyID = team.WyID
}
}
if len(r.HomeScores) > 0 {
match.HomeScore = r.HomeScores[0]
}
if len(r.AwayScores) > 0 {
match.AwayScore = r.AwayScores[0]
}
if len(r.HomeScores) > 6 {
match.HomeScorePenalties = r.HomeScores[6]
}
if len(r.AwayScores) > 6 {
match.AwayScorePenalties = r.AwayScores[6]
}
if r.Environment.Temperature != "" {
if temp, err := strconv.ParseFloat(r.Environment.Temperature, 64); err == nil {
match.Temperature = &temp
}
}
if r.Environment.Weather != 0 {
w := fmt.Sprintf("%d", r.Environment.Weather)
match.Weather = &w
}
if r.UpdatedAt > 0 {
ut := time.Unix(r.UpdatedAt, 0).UTC()
match.APILastSyncedAt = &ut
match.APISyncStatus = "synced"
}
if err := h.DB.Save(&match).Error; err != nil {
errorsCount++
continue
}
updated++
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Matches diary import completed",
"data": gin.H{
"imported": imported,
"updated": updated,
"errors": errorsCount,
},
"timestamp": time.Now().UTC().Format(time.RFC3339),
})
}
func RegisterImportRoutes(rg *gin.RouterGroup, db *gorm.DB, cfg config.Config) {
h := &ImportHandler{
DB: db,
......@@ -2019,11 +1413,7 @@ func RegisterImportRoutes(rg *gin.RouterGroup, db *gorm.DB, cfg config.Config) {
imp.POST("/teams/images", h.ImportTeamImages)
imp.POST("/coaches", h.ImportCoaches)
imp.POST("/referees", h.ImportReferees)
imp.POST("/matches/recent", h.ImportMatchesRecent)
imp.POST("/matches/diary", h.ImportMatchesDiary)
imp.POST("/matches/lineup", h.ImportMatchLineup)
imp.POST("/matches/advancedstats", h.ImportMatchAdvancedStats)
imp.POST("/matches/list", h.ImportMatchesList)
imp.POST("/matches/fixtures", h.ImportMatchesFixtures)
imp.POST("/matches/wyscout", h.ImportMatchesWyscout)
imp.POST("/matches/formations", h.ImportMatchFormations)
......@@ -2164,6 +1554,9 @@ func (h *ImportHandler) ImportMatchAdvancedStats(c *gin.Context) {
if err != nil {
return err
}
if status == http.StatusNoContent || status == http.StatusNotFound {
return nil
}
if status != http.StatusOK {
return fmt.Errorf("provider status %d: %s", status, truncate(body))
}
......@@ -2461,7 +1854,7 @@ func (h *ImportHandler) ImportMatchAdvancedStats(c *gin.Context) {
if ck.LastMatchWyID > 0 {
if err := h.DB.WithContext(ctx).
Model(&models.Match{}).
Where("wy_id IS NOT NULL AND wy_id > ?", ck.LastMatchWyID).
Where("wy_id IS NOT NULL AND wy_id > 0 AND wy_id < ?", ck.LastMatchWyID).
Count(&remaining).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to count remaining matches"})
return
......@@ -2544,6 +1937,18 @@ func (h *ImportHandler) ImportMatchAdvancedStats(c *gin.Context) {
var produced int64
nextSeq := int64(1)
localLast := ck.LastMatchWyID
if localLast == 0 {
var maxWyID int
if err := h.DB.WithContext(ctx).
Model(&models.Match{}).
Select("COALESCE(MAX(wy_id), 0)").
Where("wy_id IS NOT NULL AND wy_id > 0").
Scan(&maxWyID).Error; err != nil {
producerErr <- err
return
}
localLast = maxWyID + 1
}
for {
if limit > 0 && produced >= int64(limit) {
return
......@@ -2553,9 +1958,9 @@ func (h *ImportHandler) ImportMatchAdvancedStats(c *gin.Context) {
Model(&models.Match{}).
Select("wy_id").
Where("wy_id IS NOT NULL AND wy_id > 0").
Order("wy_id")
Order("wy_id DESC")
if localLast > 0 {
q = q.Where("wy_id > ?", localLast)
q = q.Where("wy_id < ?", localLast)
}
need := batchSize
if limit > 0 {
......@@ -6441,465 +5846,6 @@ func (h *ImportHandler) ImportMatchesWyscout(c *gin.Context) {
})
}
// ImportMatchesList imports all matches from TheSports API (match/list) for one-time full sync.
// @Summary Import all matches from TheSports (one-time full sync)
// @Description Performs a full import of all matches from TheSports match/list API using pagination. This is intended for one-time initial sync to get all historical matches. The API returns 1000 matches per page and stops when total is 0. Use startPage to resume from a specific page if the import was interrupted.
// @Tags Import
// @Param startPage query int false "Starting page number (default: 1, use to resume interrupted import)"
// @Success 200 {object} map[string]interface{}
// @Failure 500 {object} map[string]string
// @Router /import/matches/list [post]
func (h *ImportHandler) ImportMatchesList(c *gin.Context) {
user := h.Cfg.ProviderUser
secret := h.Cfg.ProviderSecret
if user == "" || secret == "" {
c.JSON(http.StatusInternalServerError, gin.H{"error": "ProviderUser/ProviderSecret not configured"})
return
}
startPageStr := c.DefaultQuery("startPage", "1")
startPage, err := strconv.Atoi(startPageStr)
if err != nil || startPage < 1 {
startPage = 1
}
totalMatches := 0
page := startPage
const pageSize = 1000 // TheSports API default
fmt.Printf("ImportMatchesList: Starting from page %d\n", startPage)
for {
url := fmt.Sprintf("https://api.thesports.com/v1/football/match/list?user=%s&secret=%s&page=%d", user, secret, page)
resp, err := h.Client.Get(url)
if err != nil {
fmt.Printf("ImportMatchesList: ERROR - failed to call TheSports API page %d: %v\n", page, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to call TheSports API: %v", err)})
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
fmt.Printf("ImportMatchesList: ERROR - provider returned status %d for page %d\n", resp.StatusCode, page)
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("provider returned status %d", resp.StatusCode)})
return
}
var payload struct {
Code int `json:"code"`
Query struct {
Total int `json:"total"`
Type string `json:"type"`
Page int `json:"page"`
} `json:"query"`
Results []struct {
ID string `json:"id"`
SeasonID string `json:"season_id"`
CompetitionID string `json:"competition_id"`
HomeTeamID string `json:"home_team_id"`
AwayTeamID string `json:"away_team_id"`
StatusID int `json:"status_id"`
MatchTime int64 `json:"match_time"`
VenueID string `json:"venue_id"`
RefereeID string `json:"referee_id"`
Neutral int `json:"neutral"`
Note string `json:"note"`
HomeScores []int `json:"home_scores"`
AwayScores []int `json:"away_scores"`
HomePosition string `json:"home_position"`
AwayPosition string `json:"away_position"`
Coverage struct {
Mlive int `json:"mlive"`
Lineup int `json:"lineup"`
} `json:"coverage"`
Round struct {
StageID string `json:"stage_id"`
GroupNum int `json:"group_num"`
RoundNum int `json:"round_num"`
} `json:"round"`
Environment struct {
Weather int `json:"weather"`
Pressure string `json:"pressure"`
Temperature string `json:"temperature"`
Wind string `json:"wind"`
Humidity string `json:"humidity"`
} `json:"environment"`
TBD int `json:"tbd"`
HasOT int `json:"has_ot"`
Ended int `json:"ended"`
TeamReverse int `json:"team_reverse"`
Loss int `json:"loss"`
UpdatedAt int64 `json:"updated_at"`
} `json:"results"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
fmt.Printf("ImportMatchesList: ERROR - failed to decode response for page %d: %v\n", page, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to decode response: %v", err)})
return
}
if payload.Code != 0 {
fmt.Printf("ImportMatchesList: ERROR - provider returned code=%d for page %d\n", payload.Code, page)
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("provider returned code %d", payload.Code)})
return
}
if len(payload.Results) == 0 {
fmt.Printf("ImportMatchesList: No more results, stopping at page %d\n", page-1)
break
}
// Process matches
for _, result := range payload.Results {
// Convert timestamp to time
matchDate := time.Unix(result.MatchTime, 0)
// Map status_id to string
status := "scheduled"
switch result.StatusID {
case 0:
status = "scheduled"
case 1:
status = "in_progress" // TheSports API uses 1 for live, map to in_progress
case 2:
status = "completed"
case 3:
status = "postponed"
case 4:
status = "cancelled"
}
// Extract scores
homeScore := 0
awayScore := 0
if len(result.HomeScores) > 0 {
homeScore = result.HomeScores[0]
}
if len(result.AwayScores) > 0 {
awayScore = result.AwayScores[0]
}
// Parse temperature if available
var temperature *float64
if result.Environment.Temperature != "" {
if temp, err := strconv.ParseFloat(result.Environment.Temperature, 64); err == nil {
temperature = &temp
}
}
// Create/update match
match := models.Match{
TsID: &result.ID,
HomeTeamTsID: &result.HomeTeamID,
AwayTeamTsID: &result.AwayTeamID,
MatchDate: matchDate,
Status: status,
HomeScore: homeScore,
AwayScore: awayScore,
Temperature: temperature,
Notes: &result.Note,
APILastSyncedAt: &time.Time{},
APISyncStatus: "synced",
}
// Map weather ID to string if available
if result.Environment.Weather != 0 {
weatherStr := fmt.Sprintf("%d", result.Environment.Weather)
match.Weather = &weatherStr
}
// Set nullable scores for penalties if available
if len(result.HomeScores) > 6 {
match.HomeScorePenalties = result.HomeScores[6]
}
if len(result.AwayScores) > 6 {
match.AwayScorePenalties = result.AwayScores[6]
}
// Upsert match
if match.TsID == nil {
continue
}
if err := h.DB.Where("ts_id = ?", *match.TsID).Assign(&match).FirstOrCreate(&match).Error; err != nil {
fmt.Printf("ImportMatchesList: ERROR - failed to upsert match %s: %v\n", *match.TsID, err)
continue
}
totalMatches++
}
fmt.Printf("ImportMatchesList: Processed page %d, got %d matches (total: %d)\n", page, len(payload.Results), totalMatches)
// Check if we should continue pagination
if payload.Query.Total == 0 || len(payload.Results) < pageSize {
fmt.Printf("ImportMatchesList: Reached end of pagination at page %d\n", page)
break
}
page++
}
fmt.Printf("ImportMatchesList: Completed full sync. Total matches imported: %d\n", totalMatches)
c.JSON(http.StatusOK, gin.H{
"message": "Full match import completed",
"totalMatches": totalMatches,
"pagesProcessed": page - 1,
})
}
// ImportMatchesRecent imports recent matches from TheSports API (match/recent/list).
// @Summary Import recent matches from TheSports
// @Description Performs a matches import using TheSports match/recent/list API. If `since` is provided (unix seconds), only matches updated since that time are fetched using the time-based endpoint. Otherwise, a full import is performed using page-based pagination (last 30 days).
// @Tags Import
// @Param pageSize query int false "Page size per request (default 100, only used for full imports)"
// @Param since query int false "Unix timestamp (seconds) to import only matches updated since this time"
// @Success 200 {object} map[string]interface{}
// @Failure 500 {object} map[string]string
// @Router /import/matches/recent [post]
func (h *ImportHandler) ImportMatchesRecent(c *gin.Context) {
user := h.Cfg.ProviderUser
secret := h.Cfg.ProviderSecret
if user == "" || secret == "" {
c.JSON(http.StatusInternalServerError, gin.H{"error": "ProviderUser/ProviderSecret not configured"})
return
}
pageSizeStr := c.DefaultQuery("pageSize", "100")
pageSize, err := strconv.Atoi(pageSizeStr)
if err != nil || pageSize <= 0 {
pageSize = 100
}
sinceStr := c.Query("since")
var since *int64
if sinceStr != "" {
if v, err := strconv.ParseInt(sinceStr, 10, 64); err == nil && v > 0 {
since = &v
}
}
imported, updated, errorsCount := 0, 0, 0
fetchAndUpsert := func(url string) error {
resp, err := h.Client.Get(url)
if err != nil {
log.Printf("failed to call TheSports matches API: %v", err)
return err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return fmt.Errorf("provider returned status %d", resp.StatusCode)
}
var payload struct {
Query struct {
Total int `json:"total"`
} `json:"query"`
Results []struct {
ID string `json:"id"`
SeasonID string `json:"season_id"`
CompetitionID string `json:"competition_id"`
HomeTeamID string `json:"home_team_id"`
AwayTeamID string `json:"away_team_id"`
StatusID int `json:"status_id"`
MatchTime int64 `json:"match_time"`
VenueID string `json:"venue_id"`
RefereeID string `json:"referee_id"`
Neutral int `json:"neutral"`
Note string `json:"note"`
HomeScores []int `json:"home_scores"`
AwayScores []int `json:"away_scores"`
HomePosition string `json:"home_position"`
AwayPosition string `json:"away_position"`
Coverage struct {
MLive int `json:"mlive"`
Lineup int `json:"lineup"`
} `json:"coverage"`
Round struct {
StageID string `json:"stage_id"`
GroupNum int `json:"group_num"`
RoundNum int `json:"round_num"`
} `json:"round"`
RelatedID string `json:"related_id"`
AggScore []int `json:"agg_score"`
Environment struct {
Weather int `json:"weather"`
Pressure string `json:"pressure"`
Temperature string `json:"temperature"`
Wind string `json:"wind"`
Humidity string `json:"humidity"`
} `json:"environment"`
TBD int `json:"tbd"`
HasOT int `json:"has_ot"`
Ended int `json:"ended"`
TeamReverse int `json:"team_reverse"`
Loss int `json:"loss"`
UpdatedAt int64 `json:"updated_at"`
} `json:"results"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
resp.Body.Close()
return err
}
resp.Body.Close()
if payload.Query.Total == 0 || len(payload.Results) == 0 {
return fmt.Errorf("no more results")
}
for _, r := range payload.Results {
var match models.Match
if err := h.DB.Where("ts_id = ?", r.ID).First(&match).Error; err != nil {
if err == gorm.ErrRecordNotFound {
tsID := r.ID
match = models.Match{
TsID: &tsID,
}
// basic mapping
if r.MatchTime > 0 {
mt := time.Unix(r.MatchTime, 0).UTC()
match.MatchDate = mt
}
if r.HomeTeamID != "" {
match.HomeTeamTsID = &r.HomeTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.HomeTeamID).First(&team).Error; err == nil {
match.HomeTeamWyID = team.WyID
}
}
if r.AwayTeamID != "" {
match.AwayTeamTsID = &r.AwayTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.AwayTeamID).First(&team).Error; err == nil {
match.AwayTeamWyID = team.WyID
}
}
// scores: index 0 is regular time goals
if len(r.HomeScores) > 0 {
match.HomeScore = r.HomeScores[0]
}
if len(r.AwayScores) > 0 {
match.AwayScore = r.AwayScores[0]
}
// penalties (index 6)
if len(r.HomeScores) > 6 {
match.HomeScorePenalties = r.HomeScores[6]
}
if len(r.AwayScores) > 6 {
match.AwayScorePenalties = r.AwayScores[6]
}
// environment temperature if present
if r.Environment.Temperature != "" {
if temp, err := strconv.ParseFloat(r.Environment.Temperature, 64); err == nil {
match.Temperature = &temp
}
}
if r.Environment.Weather != 0 {
w := fmt.Sprintf("%d", r.Environment.Weather)
match.Weather = &w
}
// updated_at metadata
if r.UpdatedAt > 0 {
ut := time.Unix(r.UpdatedAt, 0).UTC()
match.APILastSyncedAt = &ut
match.APISyncStatus = "synced"
}
if err := h.DB.Create(&match).Error; err != nil {
errorsCount++
continue
}
imported++
continue
}
errorsCount++
continue
}
// update existing match
if r.MatchTime > 0 {
mt := time.Unix(r.MatchTime, 0).UTC()
match.MatchDate = mt
}
if r.HomeTeamID != "" {
match.HomeTeamTsID = &r.HomeTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.HomeTeamID).First(&team).Error; err == nil {
match.HomeTeamWyID = team.WyID
}
}
if r.AwayTeamID != "" {
match.AwayTeamTsID = &r.AwayTeamID
var team models.Team
if err := h.DB.Where("ts_id = ?", r.AwayTeamID).First(&team).Error; err == nil {
match.AwayTeamWyID = team.WyID
}
}
if len(r.HomeScores) > 0 {
match.HomeScore = r.HomeScores[0]
}
if len(r.AwayScores) > 0 {
match.AwayScore = r.AwayScores[0]
}
if len(r.HomeScores) > 6 {
match.HomeScorePenalties = r.HomeScores[6]
}
if len(r.AwayScores) > 6 {
match.AwayScorePenalties = r.AwayScores[6]
}
if r.Environment.Temperature != "" {
if temp, err := strconv.ParseFloat(r.Environment.Temperature, 64); err == nil {
match.Temperature = &temp
}
}
if r.Environment.Weather != 0 {
w := fmt.Sprintf("%d", r.Environment.Weather)
match.Weather = &w
}
if r.UpdatedAt > 0 {
ut := time.Unix(r.UpdatedAt, 0).UTC()
match.APILastSyncedAt = &ut
match.APISyncStatus = "synced"
}
if err := h.DB.Save(&match).Error; err != nil {
errorsCount++
continue
}
updated++
}
return nil
}
if since != nil {
// incremental, time-based import
url := fmt.Sprintf("https://api.thesports.com/v1/football/match/recent/list?user=%s&secret=%s&type=time&time=%d", user, secret, *since)
_ = fetchAndUpsert(url)
} else {
// full page-based import
for page := 1; ; page++ {
url := fmt.Sprintf("https://api.thesports.com/v1/football/match/recent/list?user=%s&secret=%s&type=page&page=%d&page_size=%d", user, secret, page, pageSize)
if err := fetchAndUpsert(url); err != nil {
break
}
}
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Matches (recent) import completed",
"data": gin.H{
"imported": imported,
"updated": updated,
"errors": errorsCount,
},
"timestamp": time.Now().UTC().Format(time.RFC3339),
})
}
// ImportCompetitions imports competitions from TheSports API.
// @Summary Import competitions from TheSports
// @Description Performs a competition import using TheSports competition additional list API. If `since` is provided (unix seconds), only competitions updated since that time are fetched using the time-based endpoint. Otherwise, a full import is performed using page-based pagination.
......
internal/handlers/players.go
......@@ -263,6 +263,108 @@ func (h *PlayerHandler) GetAdvancedStatsAverages(c *gin.Context) {
}
}
var allRows []models.PlayerAdvancedStats
if err := h.DB.WithContext(c.Request.Context()).
Where("player_wy_id = ?", wyID).
Order("season_id DESC").
Find(&allRows).Error; err != nil {
respondError(c, err)
return
}
type competitionSummary struct {
WyID int `json:"wyId"`
Name *string `json:"name,omitempty"`
Logo *string `json:"logo,omitempty"`
}
type seasonSummary struct {
WyID int `json:"wyId"`
Name *string `json:"name,omitempty"`
}
type advancedStatsOut struct {
models.PlayerAdvancedStats
Season *seasonSummary `json:"season,omitempty"`
Competition *competitionSummary `json:"competition,omitempty"`
}
seasonIDs := make([]int, 0, len(allRows))
competitionIDs := make([]int, 0, len(allRows))
seenSeason := map[int]struct{}{}
seenCompetition := map[int]struct{}{}
for _, row := range allRows {
if row.SeasonID > 0 {
if _, ok := seenSeason[row.SeasonID]; !ok {
seenSeason[row.SeasonID] = struct{}{}
seasonIDs = append(seasonIDs, row.SeasonID)
}
}
if row.CompetitionID > 0 {
if _, ok := seenCompetition[row.CompetitionID]; !ok {
seenCompetition[row.CompetitionID] = struct{}{}
competitionIDs = append(competitionIDs, row.CompetitionID)
}
}
}
seasonsByWyID := map[int]seasonSummary{}
if len(seasonIDs) > 0 {
var seasons []struct {
WyID int `gorm:"column:wy_id"`
Name string `gorm:"column:name"`
}
if err := h.DB.WithContext(c.Request.Context()).
Model(&models.Season{}).
Select("wy_id", "name").
Where("wy_id IN ?", seasonIDs).
Find(&seasons).Error; err == nil {
for _, s := range seasons {
name := s.Name
seasonsByWyID[s.WyID] = seasonSummary{WyID: s.WyID, Name: &name}
}
}
}
competitionsByWyID := map[int]competitionSummary{}
if len(competitionIDs) > 0 {
var comps []struct {
WyID int `gorm:"column:wy_id"`
Name string `gorm:"column:name"`
Logo *string `gorm:"column:logo"`
}
if err := h.DB.WithContext(c.Request.Context()).
Model(&models.Competition{}).
Select("wy_id", "name", "logo").
Where("wy_id IN ?", competitionIDs).
Find(&comps).Error; err == nil {
for _, comp := range comps {
name := comp.Name
var logo *string
if comp.Logo != nil && *comp.Logo != "" {
v := *comp.Logo
logo = &v
}
competitionsByWyID[comp.WyID] = competitionSummary{WyID: comp.WyID, Name: &name, Logo: logo}
}
}
}
outItems := make([]advancedStatsOut, 0, len(allRows))
for _, row := range allRows {
var season *seasonSummary
if v, ok := seasonsByWyID[row.SeasonID]; ok {
tmp := v
season = &tmp
}
var competition *competitionSummary
if v, ok := competitionsByWyID[row.CompetitionID]; ok {
tmp := v
competition = &tmp
}
outItems = append(outItems, advancedStatsOut{PlayerAdvancedStats: row, Season: season, Competition: competition})
}
timestamp := time.Now().UTC().Format(time.RFC3339)
endpoint := fmt.Sprintf("/players/wyscout/%d/advancedstats", wyID)
......@@ -274,6 +376,7 @@ func (h *PlayerHandler) GetAdvancedStatsAverages(c *gin.Context) {
"wyId": anchorSeasonID,
"name": anchorSeasonName,
},
"items": outItems,
"seasonAvg": aggregateNumericPointers(seasonRows),
"last2YearsAvg": aggregateNumericPointers(last2Rows),
"last5YearsAvg": aggregateNumericPointers(last5Rows),
......@@ -1199,6 +1302,7 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) {
WyID int `json:"wyId"`
Name *string `json:"name,omitempty"`
Image *string `json:"image,omitempty"`
Type *string `json:"type,omitempty"`
}
type competitionSummary struct {
......@@ -1253,10 +1357,11 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) {
WyID int `gorm:"column:wy_id"`
Name string `gorm:"column:name"`
ImageDataURL *string `gorm:"column:image_data_url"`
Type string `gorm:"column:type"`
}
if err := h.DB.WithContext(c.Request.Context()).
Model(&models.Team{}).
Select("wy_id", "name", "image_data_url").
Select("wy_id", "name", "image_data_url", "type").
Where("wy_id IN ?", teamIDs).
Find(&teams).Error; err == nil {
for _, t := range teams {
......@@ -1266,7 +1371,12 @@ func (h *PlayerHandler) ListCareerByHudlID(c *gin.Context) {
v := *t.ImageDataURL
img = &v
}
teamsByWyID[t.WyID] = teamSummary{WyID: t.WyID, Name: &name, Image: img}
var typ *string
if t.Type != "" {
v := t.Type
typ = &v
}
teamsByWyID[t.WyID] = teamSummary{WyID: t.WyID, Name: &name, Image: img, Type: typ}
}
}
}
......
migrations/0017_add_player_name_search_indexes.sql
......@@ -7,6 +7,14 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm;
-- Enable unaccent extension for accent-insensitive search (if not already enabled)
CREATE EXTENSION IF NOT EXISTS unaccent;
CREATE OR REPLACE FUNCTION unaccent_immutable(text)
RETURNS text
LANGUAGE sql
IMMUTABLE
AS $$
SELECT unaccent($1);
$$;
-- Add GIN trigram indexes for fast ILIKE searches on name columns
-- These indexes dramatically speed up pattern matching queries like ILIKE '%search%'
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_short_name_trgm
......@@ -60,28 +68,28 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_short_name_trgm
-- Accent-insensitive trigram indexes (support searches where user input may omit accents)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_short_name_unaccent_trgm
ON players USING GIN (unaccent(LOWER(short_name)) gin_trgm_ops);
ON players USING GIN (unaccent_immutable(LOWER(short_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_first_name_unaccent_trgm
ON players USING GIN (unaccent(LOWER(first_name)) gin_trgm_ops);
ON players USING GIN (unaccent_immutable(LOWER(first_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_players_last_name_unaccent_trgm
ON players USING GIN (unaccent(LOWER(last_name)) gin_trgm_ops);
ON players USING GIN (unaccent_immutable(LOWER(last_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_teams_name_unaccent_trgm
ON teams USING GIN (unaccent(LOWER(name)) gin_trgm_ops);
ON teams USING GIN (unaccent_immutable(LOWER(name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_teams_short_name_unaccent_trgm
ON teams USING GIN (unaccent(LOWER(short_name)) gin_trgm_ops);
ON teams USING GIN (unaccent_immutable(LOWER(short_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_first_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(first_name)) gin_trgm_ops);
ON coaches USING GIN (unaccent_immutable(LOWER(first_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_middle_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(middle_name)) gin_trgm_ops);
ON coaches USING GIN (unaccent_immutable(LOWER(middle_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_last_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(last_name)) gin_trgm_ops);
ON coaches USING GIN (unaccent_immutable(LOWER(last_name)) gin_trgm_ops);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_coaches_short_name_unaccent_trgm
ON coaches USING GIN (unaccent(LOWER(short_name)) gin_trgm_ops);
ON coaches USING GIN (unaccent_immutable(LOWER(short_name)) gin_trgm_ops);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment