iamBadgers/RushStatistics
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

create the db file from the google doc

Browse Source
This commit is contained in:
iamBadgers
2024-06-08 21:31:07 -07:00
parent 395c06e88e
commit 7fa18b15a1
5 changed files with 231 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -1,6 +1,7 @@
.DS_Store .DS_Store
node_modules node_modules
loader/venv venv
__pycache__
# dist folders # dist folders
dist dist

14
loader/createrushdatabase.py Normal file
View File

@@ -0,0 +1,14 @@
from databasesync import createDatabase
from sheetloader import downloadGamesCSV, downloadCharactersCSV
CHARACTER_DATA_OUT_FILE = "CharacterData.csv"
GAME_DATA_OUT_FILE = "GameData.csv"
DATABASE_NAME = "testdb.db"
def execute():
downloadCharactersCSV(CHARACTER_DATA_OUT_FILE)
downloadGamesCSV(GAME_DATA_OUT_FILE)
createDatabase(DATABASE_NAME, GAME_DATA_OUT_FILE, CHARACTER_DATA_OUT_FILE)
if __name__ == "__main__":
execute()

261
loader/databasesync.py
View File

@@ -5,123 +5,192 @@ from collections import namedtuple
Character = namedtuple('Character', Character = namedtuple('Character',
['id', 'characterName', 'playerName', 'role', 'creationDate', 'status']) ['id', 'characterName', 'playerName', 'role', 'creationDate', 'status'])
Game = namedtuple('Game', Game = namedtuple('Game',
['id', 'title', 'status', 'fix', 'event', 'postdate', 'gamemaster', 'payouteb', 'payoutip', 'payoutloot']) ['id', 'title', 'status', 'fix', 'event', 'postdate', 'gamemaster', 'payouteb', 'payoutip', 'payoutloot'])
Link = namedtuple('Link', ['gameId', 'gameTitle', 'characterId', 'characterName']) Link = namedtuple('Link', ['gameId', 'gameTitle', 'characterId', 'characterName'])
APPS_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Apps" (
"gameId" INTEGER,
"gameTitle" TEXT,
"characterName" TEXT,
"characterId" INTEGER,
PRIMARY KEY("gameId","characterId"),
FOREIGN KEY("characterId") REFERENCES "Characters"("id"),
FOREIGN KEY("gameId") REFERENCES "Games"("id")
);
"""
APPS_REPLACE_INTO = """
REPLACE INTO Apps (gameId, gameTitle, characterId, characterName)
VALUES (?, ?, ?, ?)
"""
PICKS_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Picks" (
"gameId" INTEGER,
"gameTitle" TEXT,
"characterName" TEXT,
"characterId" INTEGER,
PRIMARY KEY("gameId","characterId"),
FOREIGN KEY("characterId") REFERENCES "Characters"("id"),
FOREIGN KEY("gameId") REFERENCES "Games"("id")
);
"""
PICKS_REPLACE_INTO = """
REPLACE INTO Picks (gameId, gameTitle, characterId, characterName)
VALUES (?, ?, ?, ?)
"""
GAMES_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Games" (
"id" INTEGER,
"title" TEXT,
"status" TEXT,
"fix" NUMERIC,
"postdate" INTEGER,
"gamemaster" INTEGER,
"payoutEB" INTEGER,
"payoutIP" INTEGER,
"payoutLoot" TEXT,
"event" NUMERIC,
PRIMARY KEY("id")
);
"""
GAMES_REPLACE_INTO = """
REPLACE INTO Games (id, title, status, fix, event, postdate, gamemaster, payoutEB, payoutIP, payoutLoot)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
CHARACTER_TABLE_CREATE = """
CREATE TABLE IF NOT EXISTS "Characters" (
"characterName" TEXT,
"id" INTEGER,
"playerName" TEXT,
"role" TEXT,
"creationDate" INTEGER,
"status" TEXT,
PRIMARY KEY("id")
);
"""
CHARACTERS_REPLACE_INTO = """
REPLACE INTO Characters (characterName, id, playerName, role, creationDate, status)
VALUES (?, ?, ?, ?, ?, ?)
"""
def extractCharacterFromRow(counter, row): def extractCharacterFromRow(counter, row):
return Character( return Character(
counter, counter,
row['Character'], row['Character'],
row['Player'], row['Player'],
row['Role'], row['Role'],
int(datetime.strptime(row['Approval Date'],'%m/%d/%Y').timestamp()) if row['Approval Date'] else 0, int(datetime.strptime(row['Approval Date'],'%m/%d/%Y').timestamp()) if row['Approval Date'] else 0,
row['Status']) row['Status'])
def extractGameFromRow(counter, row): def extractGameFromRow(counter, row):
return Game( return Game(
counter, counter,
row['Game Title'], row['Game Title'],
row['Game Status'], row['Game Status'],
row['Fix'], row['Fix'],
row['Event'], row['Event'],
int(datetime.strptime(row['Game Date'],'%d-%b-%Y').timestamp()) if row['Game Date'] else 0, int(datetime.strptime(row['Game Date'],'%d-%b-%Y').timestamp()) if row['Game Date'] else 0,
row['GM'], row['GM'],
row['Payout (EB)'], row['Payout (EB)'],
row['Payout (IP)'], row['Payout (IP)'],
row['Payout (Loot)']) row['Payout (Loot)'])
def loadCharacters(characterFileName): def loadCharacters(characterFileName):
characters = [] characters = []
characterNameToId = {} characterNameToId = {}
with open(characterFileName) as csvFile: with open(characterFileName) as csvFile:
reader = csv.DictReader(csvFile) reader = csv.DictReader(csvFile)
counter = 1 counter = 1
for row in reader: for row in reader:
character = extractCharacterFromRow(counter, row) character = extractCharacterFromRow(counter, row)
if character.characterName: if character.characterName:
characters.append(character) characters.append(character)
characterNameToId[character.characterName] = counter characterNameToId[character.characterName] = counter
counter += 1 counter += 1
return characters, characterNameToId return characters, characterNameToId
def loadGames(gameFileName): def loadGames(gameFileName):
games = [] games = []
gameTitleToId = {} gameTitleToId = {}
with open(gameFileName) as csvFile: with open(gameFileName) as csvFile:
reader = csv.DictReader(csvFile) reader = csv.DictReader(csvFile)
counter = 1 counter = 1
for row in reader: for row in reader:
game = extractGameFromRow(counter, row) game = extractGameFromRow(counter, row)
if game.title: if game.title:
games.append(game) games.append(game)
gameTitleToId[game.title] = counter gameTitleToId[game.title] = counter
counter += 1 counter += 1
return games, gameTitleToId return games, gameTitleToId
def loadAppsAndPicks(characterNameToId, gameTitleToId, gameFileName): def loadAppsAndPicks(characterNameToId, gameTitleToId, gameFileName):
apps = [] apps = []
picks = [] picks = []
with open(gameFileName) as csvFile: with open(gameFileName) as csvFile:
reader = csv.DictReader(csvFile) reader = csv.DictReader(csvFile)
for row in reader: for row in reader:
gameTitle = row['Game Title'] gameTitle = row['Game Title']
if (gameTitle): if (gameTitle):
gameId = gameTitleToId[gameTitle] gameId = gameTitleToId[gameTitle]
for i in range(1,7): for i in range(1,7):
characterName = row['Character {} (Player)'.format(i)] characterName = row['Character {} (Player)'.format(i)]
if characterName: if characterName:
characterId = characterNameToId[characterName] characterId = characterNameToId[characterName]
picks.append(Link(gameId, gameTitle, characterId, characterName)) picks.append(Link(gameId, gameTitle, characterId, characterName))
for i in range(7,11): for i in range(7,11):
characterName = row['Character {}'.format(i)] characterName = row['Character {}'.format(i)]
if (characterName): if (characterName):
characterId = characterNameToId[characterName] characterId = characterNameToId[characterName]
picks.append(Link(gameId, gameTitle, characterId, characterName)) picks.append(Link(gameId, gameTitle, characterId, characterName))
for i in range(1,51): for i in range(1,51):
characterName = row['Applicant {}'.format(i)] characterName = row['Applicant {}'.format(i)]
if characterName and characterName in characterNameToId: if characterName and characterName in characterNameToId:
characterId = characterNameToId[characterName] characterId = characterNameToId[characterName]
apps.append(Link(gameId, gameTitle, characterId, characterName)) apps.append(Link(gameId, gameTitle, characterId, characterName))
return apps, picks return apps, picks
def main(): def createTables(dbName):
characters, characterNameToId = loadCharacters("Character Data.csv") with sqlite3.connect(dbName) as connection:
games, gameTitleToId = loadGames("Game Data.csv") cursor = connection.cursor()
apps, picks = loadAppsAndPicks(characterNameToId, gameTitleToId, "Game Data.csv") cursor.execute(CHARACTER_TABLE_CREATE)
cursor.execute(GAMES_TABLE_CREATE)
cursor.execute(APPS_TABLE_CREATE)
cursor.execute(PICKS_TABLE_CREATE)
with sqlite3.connect('testdb.db') as connection: def loadData(dbName, gamesDataFileName, characterDataFileName):
cursor = connection.cursor() characters, characterNameToId = loadCharacters(characterDataFileName)
for character in characters: games, gameTitleToId = loadGames(gamesDataFileName)
cursor.execute(""" apps, picks = loadAppsAndPicks(characterNameToId, gameTitleToId, gamesDataFileName)
REPLACE INTO Characters (characterName, id, playerName, role, creationDate, status)
VALUES (?, ?, ?, ?, ?, ?)
""", [character.characterName, character.id, character.playerName, character.role, character.creationDate, character.status])
for game in games:
cursor.execute("""
REPLACE INTO Games (id, title, status, fix, event, postdate, gamemaster, payoutEB, payoutIP, payoutLoot)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", [game.id, game.title, game.status, game.fix, game.event, game.postdate, game.gamemaster, game.payouteb, game.payoutip, game.payoutloot])
for app in apps:
cursor.execute("""
REPLACE INTO Apps (gameId, gameTitle, characterId, characterName)
VALUES (?, ?, ?, ?)
""", [app.gameId, app.gameTitle, app.characterId, app.characterName])
for pick in picks:
cursor.execute("""
REPLACE INTO Picks (gameId, gameTitle, characterId, characterName)
VALUES (?, ?, ?, ?)
""", [pick.gameId, pick.gameTitle, pick.characterId, pick.characterName])
with sqlite3.connect(dbName) as connection:
cursor = connection.cursor()
for character in characters:
cursor.execute(CHARACTERS_REPLACE_INTO, [character.characterName, character.id, character.playerName, character.role, character.creationDate, character.status])
for game in games:
cursor.execute(GAMES_REPLACE_INTO, [game.id, game.title, game.status, game.fix, game.event, game.postdate, game.gamemaster, game.payouteb, game.payoutip, game.payoutloot])
for app in apps:
cursor.execute(APPS_REPLACE_INTO, [app.gameId, app.gameTitle, app.characterId, app.characterName])
for pick in picks:
cursor.execute(PICKS_REPLACE_INTO, [pick.gameId, pick.gameTitle, pick.characterId, pick.characterName])
if __name__ == "__main__": def createDatabase(dbName, gamesDataFileName, characterDataFileName):
main() createTables(dbName)
loadData(dbName, gamesDataFileName, characterDataFileName)

19
loader/requirements.txt Normal file
View File

@@ -0,0 +1,19 @@
cachetools==5.3.3
certifi==2024.2.2
charset-normalizer==3.3.2
google-api-core==2.19.0
google-api-python-client==2.130.0
google-auth==2.29.0
google-auth-httplib2==0.2.0
googleapis-common-protos==1.63.0
httplib2==0.22.0
idna==3.7
proto-plus==1.23.0
protobuf==4.25.3
pyasn1==0.6.0
pyasn1_modules==0.4.0
pyparsing==3.1.2
requests==2.32.2
rsa==4.9
uritemplate==4.1.1
urllib3==2.2.1

31
loader/sheetloader.py
View File

@@ -0,0 +1,31 @@
import os
import requests
import sys
DOCUMENT_ID = '1VKujaowUSxB9SuBdMt81aFlUGWo2fUCAOzX2PjYQVxs'
GAME_SHEET_ID = '160661246'
CHARACTTER_SHEET_ID = '1445780435'
def getGoogleSheet(spreadsheetId, sheetId, outFile):
url = f'https://docs.google.com/spreadsheets/d/{spreadsheetId}/export?gid={sheetId}&format=csv'
response = requests.get(url)
if response.status_code == 200:
filepath = os.path.join('./', outFile)
with open(filepath, 'wb') as f:
f.write(response.content)
print('CSV file saved to: {}'.format(filepath))
else:
print(f'Error downloading sheet: {response.status_code}')
sys.exit(1)
def downloadGamesCSV(outFile):
getGoogleSheet(DOCUMENT_ID, GAME_SHEET_ID, outFile)
def downloadCharactersCSV(outFile):
getGoogleSheet(DOCUMENT_ID, CHARACTTER_SHEET_ID, outFile)
if __name__ == "__main__":
getGoogleSheet('1VKujaowUSxB9SuBdMt81aFlUGWo2fUCAOzX2PjYQVxs', '1445780435', './here', 'potato')