RushStatistics/loader/databasesync.py

import csv
import sqlite3
from datetime import datetime
from collections import namedtuple


Character = namedtuple('Character', 
  ['id', 'characterName', 'playerName', 'role', 'creationDate', 'status'])

Game = namedtuple('Game',
  ['id', 'title', 'status', 'fix', 'event', 'postdate', 'gamemaster', 'payouteb', 'payoutip', 'payoutloot'])

Link = namedtuple('Link', ['gameId', 'gameTitle', 'characterId', 'characterName'])

SET_WAL_PRAGMA = """
PRAGMA journal_mode=WAL;
"""

APPS_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Apps" (
  "gameId"  INTEGER,
  "gameTitle" TEXT,
  "characterName" TEXT,
  "characterId" INTEGER,
  PRIMARY KEY("gameId","characterId"),
  FOREIGN KEY("characterId") REFERENCES "Characters"("id"),
  FOREIGN KEY("gameId") REFERENCES "Games"("id")
);
"""

APPS_REPLACE_INTO = """
REPLACE INTO Apps (gameId, gameTitle, characterId, characterName)
VALUES (?, ?, ?, ?)
"""


PICKS_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Picks" (
  "gameId"  INTEGER,
  "gameTitle" TEXT,
  "characterName" TEXT,
  "characterId" INTEGER,
  PRIMARY KEY("gameId","characterId"),
  FOREIGN KEY("characterId") REFERENCES "Characters"("id"),
  FOREIGN KEY("gameId") REFERENCES "Games"("id")
);
"""

PICKS_REPLACE_INTO = """
REPLACE INTO Picks (gameId, gameTitle, characterId, characterName)
VALUES (?, ?, ?, ?)
"""

GAMES_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Games" (
  "id"  INTEGER,
  "title" TEXT,
  "status"  TEXT,
  "fix" NUMERIC,
  "postdate"  INTEGER,
  "gamemaster"  INTEGER,
  "payoutEB"  INTEGER,
  "payoutIP"  INTEGER,
  "payoutLoot"  TEXT,
  "event" NUMERIC,
  PRIMARY KEY("id")
);
"""

GAMES_REPLACE_INTO = """
REPLACE INTO Games (id, title, status, fix, event, postdate, gamemaster, payoutEB, payoutIP, payoutLoot)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""

CHARACTER_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Characters" (
  "characterName" TEXT,
  "id"  INTEGER,
  "playerName"  TEXT,
  "role"  TEXT,
  "creationDate"  INTEGER,
  "status"  TEXT,
  PRIMARY KEY("id")
);
"""

CHARACTERS_REPLACE_INTO = """
REPLACE INTO Characters (characterName, id, playerName, role, creationDate, status)
VALUES (?, ?, ?, ?, ?, ?)
"""

def extractCharacterFromRow(counter, row):
  return Character(
    counter,
    row['Character'],
    row['Player'],
    row['Role'],
    int(datetime.strptime(row['Approval Date'],'%m/%d/%Y').timestamp()) if row['Approval Date'] else 0,
    row['Status'])

def extractGameFromRow(counter, row):
  return Game(
    counter,
    row['Game Title'],
    row['Game Status'],
    row['Fix'],
    row['Event'],
    int(datetime.strptime(row['Game Date'],'%d-%b-%Y').timestamp()) if row['Game Date'] else 0,
    row['GM'],
    row['Payout (EB)'],
    row['Payout (IP)'],
    row['Payout (Loot)'])

def loadCharacters(characterFileName):
  characters = []
  characterNameToId = {}

  with open(characterFileName) as csvFile:
    reader = csv.DictReader(csvFile)
    counter = 1
    for row in reader:
      character = extractCharacterFromRow(counter, row)
      if character.characterName:
        characters.append(character)
        characterNameToId[character.characterName] = counter
        counter += 1

  return characters, characterNameToId

def loadGames(gameFileName):
  games = []
  gameTitleToId = {}

  with open(gameFileName) as csvFile:
    reader = csv.DictReader(csvFile)
    counter = 1
    for row in reader:
      game = extractGameFromRow(counter, row)
      if game.title:
        games.append(game)
        gameTitleToId[game.title] = counter
        counter += 1

  return games, gameTitleToId

def loadAppsAndPicks(characterNameToId, gameTitleToId, gameFileName):
  apps = []
  picks = []

  with open(gameFileName) as csvFile:
    reader = csv.DictReader(csvFile)
    for row in reader:
      gameTitle = row['Game Title']
      if (gameTitle):
        gameId = gameTitleToId[gameTitle]
        for i in range(1,7):
          characterName = row['Character {} (Player)'.format(i)]
          if characterName:
            characterId = characterNameToId[characterName]
            picks.append(Link(gameId, gameTitle, characterId, characterName))
        for i in range(7,11):
          characterName = row['Character {}'.format(i)]
          if (characterName):
            characterId = characterNameToId[characterName]
            picks.append(Link(gameId, gameTitle, characterId, characterName))
        for i in range(1,51):
          characterName = row['Applicant {}'.format(i)]
          if characterName and characterName in characterNameToId:
            characterId = characterNameToId[characterName]
            apps.append(Link(gameId, gameTitle, characterId, characterName))

  return apps, picks

def createTables(dbName):
  with sqlite3.connect(dbName) as connection:
    cursor = connection.cursor()
    cursor.execute(SET_WAL_PRAGMA)
    cursor.execute(CHARACTER_TABLE_CREATE)
    cursor.execute(GAMES_TABLE_CREATE)
    cursor.execute(APPS_TABLE_CREATE)
    cursor.execute(PICKS_TABLE_CREATE)

def loadData(dbName, gamesDataFileName, characterDataFileName):
  characters, characterNameToId = loadCharacters(characterDataFileName)
  games, gameTitleToId = loadGames(gamesDataFileName)
  apps, picks = loadAppsAndPicks(characterNameToId, gameTitleToId, gamesDataFileName)

  with sqlite3.connect(dbName) as connection:
    cursor = connection.cursor()
    for character in characters:
      cursor.execute(CHARACTERS_REPLACE_INTO, [character.characterName, character.id, character.playerName, character.role, character.creationDate, character.status])
    for game in games:
      cursor.execute(GAMES_REPLACE_INTO, [game.id, game.title, game.status, game.fix, game.event, game.postdate, game.gamemaster, game.payouteb, game.payoutip, game.payoutloot])
    for app in apps:
      cursor.execute(APPS_REPLACE_INTO, [app.gameId, app.gameTitle, app.characterId, app.characterName])
    for pick in picks:
      cursor.execute(PICKS_REPLACE_INTO, [pick.gameId, pick.gameTitle, pick.characterId, pick.characterName])

def createDatabase(dbName, gamesDataFileName, characterDataFileName):
  createTables(dbName)
  loadData(dbName, gamesDataFileName, characterDataFileName)