From 7fa18b15a12b391cdae8d6a01842aab62e37e9f6 Mon Sep 17 00:00:00 2001 From: iamBadgers Date: Sat, 8 Jun 2024 21:31:07 -0700 Subject: [PATCH] create the db file from the google doc --- .gitignore | 3 +- loader/createrushdatabase.py | 14 ++ loader/databasesync.py | 261 ++++++++++++++++++++++------------- loader/requirements.txt | 19 +++ loader/sheetloader.py | 31 +++++ 5 files changed, 231 insertions(+), 97 deletions(-) create mode 100644 loader/createrushdatabase.py create mode 100644 loader/requirements.txt diff --git a/.gitignore b/.gitignore index 40bc30c..84b8d60 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store node_modules -loader/venv +venv +__pycache__ # dist folders dist diff --git a/loader/createrushdatabase.py b/loader/createrushdatabase.py new file mode 100644 index 0000000..f1d223f --- /dev/null +++ b/loader/createrushdatabase.py @@ -0,0 +1,14 @@ +from databasesync import createDatabase +from sheetloader import downloadGamesCSV, downloadCharactersCSV + +CHARACTER_DATA_OUT_FILE = "CharacterData.csv" +GAME_DATA_OUT_FILE = "GameData.csv" +DATABASE_NAME = "testdb.db" + +def execute(): + downloadCharactersCSV(CHARACTER_DATA_OUT_FILE) + downloadGamesCSV(GAME_DATA_OUT_FILE) + createDatabase(DATABASE_NAME, GAME_DATA_OUT_FILE, CHARACTER_DATA_OUT_FILE) + +if __name__ == "__main__": + execute() diff --git a/loader/databasesync.py b/loader/databasesync.py index 8e30921..41cb9a6 100644 --- a/loader/databasesync.py +++ b/loader/databasesync.py @@ -5,123 +5,192 @@ from collections import namedtuple Character = namedtuple('Character', - ['id', 'characterName', 'playerName', 'role', 'creationDate', 'status']) + ['id', 'characterName', 'playerName', 'role', 'creationDate', 'status']) Game = namedtuple('Game', - ['id', 'title', 'status', 'fix', 'event', 'postdate', 'gamemaster', 'payouteb', 'payoutip', 'payoutloot']) + ['id', 'title', 'status', 'fix', 'event', 'postdate', 'gamemaster', 'payouteb', 'payoutip', 'payoutloot']) Link = namedtuple('Link', ['gameId', 'gameTitle', 'characterId', 'characterName']) +APPS_TABLE_CREATE = """ +CREATE TABLE IF NOT EXISTS "Apps" ( + "gameId" INTEGER, + "gameTitle" TEXT, + "characterName" TEXT, + "characterId" INTEGER, + PRIMARY KEY("gameId","characterId"), + FOREIGN KEY("characterId") REFERENCES "Characters"("id"), + FOREIGN KEY("gameId") REFERENCES "Games"("id") +); +""" + +APPS_REPLACE_INTO = """ +REPLACE INTO Apps (gameId, gameTitle, characterId, characterName) +VALUES (?, ?, ?, ?) +""" + + +PICKS_TABLE_CREATE = """ +CREATE TABLE IF NOT EXISTS "Picks" ( + "gameId" INTEGER, + "gameTitle" TEXT, + "characterName" TEXT, + "characterId" INTEGER, + PRIMARY KEY("gameId","characterId"), + FOREIGN KEY("characterId") REFERENCES "Characters"("id"), + FOREIGN KEY("gameId") REFERENCES "Games"("id") +); +""" + +PICKS_REPLACE_INTO = """ +REPLACE INTO Picks (gameId, gameTitle, characterId, characterName) +VALUES (?, ?, ?, ?) +""" + +GAMES_TABLE_CREATE = """ +CREATE TABLE IF NOT EXISTS "Games" ( + "id" INTEGER, + "title" TEXT, + "status" TEXT, + "fix" NUMERIC, + "postdate" INTEGER, + "gamemaster" INTEGER, + "payoutEB" INTEGER, + "payoutIP" INTEGER, + "payoutLoot" TEXT, + "event" NUMERIC, + PRIMARY KEY("id") +); +""" + +GAMES_REPLACE_INTO = """ +REPLACE INTO Games (id, title, status, fix, event, postdate, gamemaster, payoutEB, payoutIP, payoutLoot) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" + +CHARACTER_TABLE_CREATE = """ +CREATE TABLE IF NOT EXISTS "Characters" ( + "characterName" TEXT, + "id" INTEGER, + "playerName" TEXT, + "role" TEXT, + "creationDate" INTEGER, + "status" TEXT, + PRIMARY KEY("id") +); +""" + +CHARACTERS_REPLACE_INTO = """ +REPLACE INTO Characters (characterName, id, playerName, role, creationDate, status) +VALUES (?, ?, ?, ?, ?, ?) +""" + def extractCharacterFromRow(counter, row): - return Character( - counter, - row['Character'], - row['Player'], - row['Role'], - int(datetime.strptime(row['Approval Date'],'%m/%d/%Y').timestamp()) if row['Approval Date'] else 0, - row['Status']) + return Character( + counter, + row['Character'], + row['Player'], + row['Role'], + int(datetime.strptime(row['Approval Date'],'%m/%d/%Y').timestamp()) if row['Approval Date'] else 0, + row['Status']) def extractGameFromRow(counter, row): - return Game( - counter, - row['Game Title'], - row['Game Status'], - row['Fix'], - row['Event'], - int(datetime.strptime(row['Game Date'],'%d-%b-%Y').timestamp()) if row['Game Date'] else 0, - row['GM'], - row['Payout (EB)'], - row['Payout (IP)'], - row['Payout (Loot)']) + return Game( + counter, + row['Game Title'], + row['Game Status'], + row['Fix'], + row['Event'], + int(datetime.strptime(row['Game Date'],'%d-%b-%Y').timestamp()) if row['Game Date'] else 0, + row['GM'], + row['Payout (EB)'], + row['Payout (IP)'], + row['Payout (Loot)']) def loadCharacters(characterFileName): - characters = [] - characterNameToId = {} + characters = [] + characterNameToId = {} - with open(characterFileName) as csvFile: - reader = csv.DictReader(csvFile) - counter = 1 - for row in reader: - character = extractCharacterFromRow(counter, row) - if character.characterName: - characters.append(character) - characterNameToId[character.characterName] = counter - counter += 1 + with open(characterFileName) as csvFile: + reader = csv.DictReader(csvFile) + counter = 1 + for row in reader: + character = extractCharacterFromRow(counter, row) + if character.characterName: + characters.append(character) + characterNameToId[character.characterName] = counter + counter += 1 - return characters, characterNameToId + return characters, characterNameToId def loadGames(gameFileName): - games = [] - gameTitleToId = {} + games = [] + gameTitleToId = {} - with open(gameFileName) as csvFile: - reader = csv.DictReader(csvFile) - counter = 1 - for row in reader: - game = extractGameFromRow(counter, row) - if game.title: - games.append(game) - gameTitleToId[game.title] = counter - counter += 1 + with open(gameFileName) as csvFile: + reader = csv.DictReader(csvFile) + counter = 1 + for row in reader: + game = extractGameFromRow(counter, row) + if game.title: + games.append(game) + gameTitleToId[game.title] = counter + counter += 1 - return games, gameTitleToId + return games, gameTitleToId def loadAppsAndPicks(characterNameToId, gameTitleToId, gameFileName): - apps = [] - picks = [] + apps = [] + picks = [] - with open(gameFileName) as csvFile: - reader = csv.DictReader(csvFile) - for row in reader: - gameTitle = row['Game Title'] - if (gameTitle): - gameId = gameTitleToId[gameTitle] - for i in range(1,7): - characterName = row['Character {} (Player)'.format(i)] - if characterName: - characterId = characterNameToId[characterName] - picks.append(Link(gameId, gameTitle, characterId, characterName)) - for i in range(7,11): - characterName = row['Character {}'.format(i)] - if (characterName): - characterId = characterNameToId[characterName] - picks.append(Link(gameId, gameTitle, characterId, characterName)) - for i in range(1,51): - characterName = row['Applicant {}'.format(i)] - if characterName and characterName in characterNameToId: - characterId = characterNameToId[characterName] - apps.append(Link(gameId, gameTitle, characterId, characterName)) + with open(gameFileName) as csvFile: + reader = csv.DictReader(csvFile) + for row in reader: + gameTitle = row['Game Title'] + if (gameTitle): + gameId = gameTitleToId[gameTitle] + for i in range(1,7): + characterName = row['Character {} (Player)'.format(i)] + if characterName: + characterId = characterNameToId[characterName] + picks.append(Link(gameId, gameTitle, characterId, characterName)) + for i in range(7,11): + characterName = row['Character {}'.format(i)] + if (characterName): + characterId = characterNameToId[characterName] + picks.append(Link(gameId, gameTitle, characterId, characterName)) + for i in range(1,51): + characterName = row['Applicant {}'.format(i)] + if characterName and characterName in characterNameToId: + characterId = characterNameToId[characterName] + apps.append(Link(gameId, gameTitle, characterId, characterName)) - return apps, picks + return apps, picks -def main(): - characters, characterNameToId = loadCharacters("Character Data.csv") - games, gameTitleToId = loadGames("Game Data.csv") - apps, picks = loadAppsAndPicks(characterNameToId, gameTitleToId, "Game Data.csv") +def createTables(dbName): + with sqlite3.connect(dbName) as connection: + cursor = connection.cursor() + cursor.execute(CHARACTER_TABLE_CREATE) + cursor.execute(GAMES_TABLE_CREATE) + cursor.execute(APPS_TABLE_CREATE) + cursor.execute(PICKS_TABLE_CREATE) - with sqlite3.connect('testdb.db') as connection: - cursor = connection.cursor() - for character in characters: - cursor.execute(""" - REPLACE INTO Characters (characterName, id, playerName, role, creationDate, status) - VALUES (?, ?, ?, ?, ?, ?) - """, [character.characterName, character.id, character.playerName, character.role, character.creationDate, character.status]) - for game in games: - cursor.execute(""" - REPLACE INTO Games (id, title, status, fix, event, postdate, gamemaster, payoutEB, payoutIP, payoutLoot) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, [game.id, game.title, game.status, game.fix, game.event, game.postdate, game.gamemaster, game.payouteb, game.payoutip, game.payoutloot]) - for app in apps: - cursor.execute(""" - REPLACE INTO Apps (gameId, gameTitle, characterId, characterName) - VALUES (?, ?, ?, ?) - """, [app.gameId, app.gameTitle, app.characterId, app.characterName]) - for pick in picks: - cursor.execute(""" - REPLACE INTO Picks (gameId, gameTitle, characterId, characterName) - VALUES (?, ?, ?, ?) - """, [pick.gameId, pick.gameTitle, pick.characterId, pick.characterName]) +def loadData(dbName, gamesDataFileName, characterDataFileName): + characters, characterNameToId = loadCharacters(characterDataFileName) + games, gameTitleToId = loadGames(gamesDataFileName) + apps, picks = loadAppsAndPicks(characterNameToId, gameTitleToId, gamesDataFileName) + with sqlite3.connect(dbName) as connection: + cursor = connection.cursor() + for character in characters: + cursor.execute(CHARACTERS_REPLACE_INTO, [character.characterName, character.id, character.playerName, character.role, character.creationDate, character.status]) + for game in games: + cursor.execute(GAMES_REPLACE_INTO, [game.id, game.title, game.status, game.fix, game.event, game.postdate, game.gamemaster, game.payouteb, game.payoutip, game.payoutloot]) + for app in apps: + cursor.execute(APPS_REPLACE_INTO, [app.gameId, app.gameTitle, app.characterId, app.characterName]) + for pick in picks: + cursor.execute(PICKS_REPLACE_INTO, [pick.gameId, pick.gameTitle, pick.characterId, pick.characterName]) -if __name__ == "__main__": - main() +def createDatabase(dbName, gamesDataFileName, characterDataFileName): + createTables(dbName) + loadData(dbName, gamesDataFileName, characterDataFileName) diff --git a/loader/requirements.txt b/loader/requirements.txt new file mode 100644 index 0000000..b9e6250 --- /dev/null +++ b/loader/requirements.txt @@ -0,0 +1,19 @@ +cachetools==5.3.3 +certifi==2024.2.2 +charset-normalizer==3.3.2 +google-api-core==2.19.0 +google-api-python-client==2.130.0 +google-auth==2.29.0 +google-auth-httplib2==0.2.0 +googleapis-common-protos==1.63.0 +httplib2==0.22.0 +idna==3.7 +proto-plus==1.23.0 +protobuf==4.25.3 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyparsing==3.1.2 +requests==2.32.2 +rsa==4.9 +uritemplate==4.1.1 +urllib3==2.2.1 diff --git a/loader/sheetloader.py b/loader/sheetloader.py index e69de29..7189d9b 100644 --- a/loader/sheetloader.py +++ b/loader/sheetloader.py @@ -0,0 +1,31 @@ +import os +import requests +import sys + +DOCUMENT_ID = '1VKujaowUSxB9SuBdMt81aFlUGWo2fUCAOzX2PjYQVxs' + +GAME_SHEET_ID = '160661246' + +CHARACTTER_SHEET_ID = '1445780435' + +def getGoogleSheet(spreadsheetId, sheetId, outFile): + url = f'https://docs.google.com/spreadsheets/d/{spreadsheetId}/export?gid={sheetId}&format=csv' + response = requests.get(url) + if response.status_code == 200: + filepath = os.path.join('./', outFile) + with open(filepath, 'wb') as f: + f.write(response.content) + print('CSV file saved to: {}'.format(filepath)) + else: + print(f'Error downloading sheet: {response.status_code}') + sys.exit(1) + +def downloadGamesCSV(outFile): + getGoogleSheet(DOCUMENT_ID, GAME_SHEET_ID, outFile) + +def downloadCharactersCSV(outFile): + getGoogleSheet(DOCUMENT_ID, CHARACTTER_SHEET_ID, outFile) + + +if __name__ == "__main__": + getGoogleSheet('1VKujaowUSxB9SuBdMt81aFlUGWo2fUCAOzX2PjYQVxs', '1445780435', './here', 'potato')