From dd3c61dc6c8f0e2c8995968dda4e369b7e54bccd Mon Sep 17 00:00:00 2001 From: 0ceanSlim Date: Tue, 14 Nov 2023 09:46:21 -0500 Subject: [PATCH] pulling all info from a new source, new data sheet with all relevant card information. --- README.md | 11 +++++---- card_info.csv => data.csv | 0 pull.py => getData.py | 10 ++++---- mian.py | 48 --------------------------------------- saveImages.py | 38 +++++++++++++++++++++++++++++++ src/lorcana_1.1.py | 25 -------------------- src/lorcana_1.py | 25 -------------------- src/lorcana_2.py | 27 ---------------------- src/lorcana_promos.py | 25 -------------------- 9 files changed, 50 insertions(+), 159 deletions(-) rename card_info.csv => data.csv (100%) rename pull.py => getData.py (80%) delete mode 100644 mian.py create mode 100644 saveImages.py delete mode 100644 src/lorcana_1.1.py delete mode 100644 src/lorcana_1.py delete mode 100644 src/lorcana_2.py delete mode 100644 src/lorcana_promos.py diff --git a/README.md b/README.md index 0a2ac1f..dfbf19f 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,12 @@ cd lorcana # Usage -- The only funtionality at the moment is grabbing the pngs for all the cards in the game and saving them into a local folder. You can do that just by running the main script in the root of this directory +```bash +python getData +``` +- This retrieves all the card data from lorcania.com ```bash -python main.py -``` -This currently runs all the scripts in the src folder and grabs all current card images for the game. \ No newline at end of file +python getData +``` +- This saves all the images specified in the data.csv to a local folder \ No newline at end of file diff --git a/card_info.csv b/data.csv similarity index 100% rename from card_info.csv rename to data.csv diff --git a/pull.py b/getData.py similarity index 80% rename from pull.py rename to getData.py index da3c449..14d2d4d 100644 --- a/pull.py +++ b/getData.py @@ -26,20 +26,20 @@ for card_id in range(1, 449): data_json = json.loads(data_page) # Extract card information - card_info = data_json['props']['card'] + data_draft = data_json['props']['card'] # Define headers for CSV file if card_id == 1: # Write headers only for the first card - headers = list(card_info.keys()) - with open('card_info.csv', mode='w', newline='', encoding='utf-8') as file: + headers = list(data_draft.keys()) + with open('data_draft.csv', mode='w', newline='', encoding='utf-8') as file: writer = csv.writer(file) writer.writerow(headers) # Define rows for CSV file - rows = [list(card_info.values())] + rows = [list(data_draft.values())] # Append data to CSV file - with open('card_info.csv', mode='a', newline='', encoding='utf-8') as file: + with open('data_draft.csv', mode='a', newline='', encoding='utf-8') as file: writer = csv.writer(file) writer.writerows(rows) diff --git a/mian.py b/mian.py deleted file mode 100644 index 2c9121e..0000000 --- a/mian.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import subprocess -import sys - -def check_python_installation(): - try: - python_version = subprocess.check_output( - ["python", "--version"], stderr=subprocess.STDOUT, universal_newlines=True - ) - print(f"{python_version}") - return True - except FileNotFoundError: - return False - -def install_python(): - print("Python is not installed. Please download and install it from:") - print("https://www.python.org/downloads/") - input("Press Enter to continue after installing Python...") - sys.exit(1) - -def run_scripts_in_folder(folder_path): - if not check_python_installation(): - install_python() - - # Get a list of all files in the folder - script_files = [f for f in os.listdir(folder_path) if f.endswith('.py')] - - if not script_files: - print(f"No Python scripts found in {folder_path}") - return - - for script_file in script_files: - script_path = os.path.join(folder_path, script_file) - - # Run the script using subprocess - try: - subprocess.run(['python', script_path], check=True) - print(f"Script {script_file} executed successfully.") - except subprocess.CalledProcessError as e: - print(f"Error executing script {script_file}: {e}") - -if __name__ == "__main__": - # Specify the folder containing the scripts - scripts_folder = "src" - - run_scripts_in_folder(scripts_folder) - - print("Done.") diff --git a/saveImages.py b/saveImages.py new file mode 100644 index 0000000..e0c5f6f --- /dev/null +++ b/saveImages.py @@ -0,0 +1,38 @@ +import os +import pandas as pd +import requests +import re + +# Function to clean filename by removing problematic characters +def clean_filename(filename): + return re.sub(r'[\\/*?:"<>|]', '', filename) + +# Read the CSV file +data = pd.read_csv('data.csv') + +# Create a folder named 'images' if it doesn't exist +images_folder = 'images' +os.makedirs(images_folder, exist_ok=True) + +# Iterate through the rows +for index, row in data.iterrows(): + card_set_id = row['card_set_id'] + card_id = row['id'] + name = row['name'] + image_url = row['image'] + + # Create a folder for the card_set_id if it doesn't exist + card_set_folder = os.path.join(images_folder, str(card_set_id)) + os.makedirs(card_set_folder, exist_ok=True) + + # Get the image + response = requests.get(image_url) + if response.status_code == 200: + # Clean the filename + image_filename = clean_filename(f"{card_id} - {name}.webp") # Adjust the extension if needed + image_path = os.path.join(card_set_folder, image_filename) + with open(image_path, 'wb') as f: + f.write(response.content) + print(f"Image saved: {image_filename}") + else: + print(f"Failed to download image for {name}") diff --git a/src/lorcana_1.1.py b/src/lorcana_1.1.py deleted file mode 100644 index 11cd277..0000000 --- a/src/lorcana_1.1.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import requests - -def download_images(base_url, set_name, num_images, local_folder): - # Ensure the local folder exists - os.makedirs(local_folder, exist_ok=True) - - for i in range(205, num_images + 1): - image_url = f"{base_url}/{set_name}/{i}.png" #might have to add the set tag here - response = requests.get(image_url) - - if response.status_code == 200: - with open(os.path.join(local_folder, f"{i}.png"), 'wb') as file: - file.write(response.content) - print(f"Downloaded {i}.png") - else: - print(f"Failed to download {i}.png - Status code: {response.status_code}") - -if __name__ == "__main__": - base_url = "https://www.lorcanawiz.com/images" #/thefirstchapter/TFC-205.png - set_name = "thefirstchapter" - num_images = 216 - local_folder = f"images/{set_name}/" - - download_images(base_url, set_name, num_images, local_folder) diff --git a/src/lorcana_1.py b/src/lorcana_1.py deleted file mode 100644 index ebf58bc..0000000 --- a/src/lorcana_1.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import requests - -def download_images(base_url, set_name, num_images, local_folder): - # Ensure the local folder exists - os.makedirs(local_folder, exist_ok=True) - - for i in range(1, num_images + 1): - image_url = f"{base_url}/{set_name}/TFC-{i}.png" #might have to add the set tag here - response = requests.get(image_url) - - if response.status_code == 200: - with open(os.path.join(local_folder, f"{i}.png"), 'wb') as file: - file.write(response.content) - print(f"Downloaded {i}.png") - else: - print(f"Failed to download {i}.png - Status code: {response.status_code}") - -if __name__ == "__main__": - base_url = "https://www.lorcanawiz.com/images" #/thefirstchapter/TFC-205.png - set_name = "thefirstchapter" - num_images = 204 - local_folder = f"images/{set_name}/" - - download_images(base_url, set_name, num_images, local_folder) diff --git a/src/lorcana_2.py b/src/lorcana_2.py deleted file mode 100644 index dbd6024..0000000 --- a/src/lorcana_2.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import requests - -def download_images(base_url, set_name, num_images, local_folder): - # Ensure the local folder exists - os.makedirs(local_folder, exist_ok=True) - - for i in range(1, num_images + 1): - image_url = f"{base_url}/{set_name}/{i}.png" - response = requests.get(image_url) - - if response.status_code == 200 and 'image' in response.headers.get('Content-Type', ''): - with open(os.path.join(local_folder, f"{i}.png"), 'wb') as file: - file.write(response.content) - print(f"Downloaded {i}.png") - elif response.status_code == 404: - print(f"Image {i}.png not found - Status code: {response.status_code}") - else: - print(f"Failed to download {i}.png - Status code: {response.status_code}") - -if __name__ == "__main__": - base_url = "https://www.lorcanawiz.com/images" - set_name = "riseofthefloodborn" - num_images = 250 - local_folder = f"images/{set_name}/" - - download_images(base_url, set_name, num_images, local_folder) diff --git a/src/lorcana_promos.py b/src/lorcana_promos.py deleted file mode 100644 index b549ddd..0000000 --- a/src/lorcana_promos.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import requests - -def download_images(base_url, set_name, num_images, local_folder): - # Ensure the local folder exists - os.makedirs(local_folder, exist_ok=True) - - for i in range(1, num_images + 1): - image_url = f"{base_url}/{set_name}/{i}.png" #might have to add the set tag here - response = requests.get(image_url) - - if response.status_code == 200 and 'image' in response.headers.get('Content-Type', ''): - with open(os.path.join(local_folder, f"{i}.png"), 'wb') as file: - file.write(response.content) - print(f"Downloaded {i}.png") - else: - print(f"Failed to download {i}.png - Status code: {response.status_code}") - -if __name__ == "__main__": - base_url = "https://www.lorcanawiz.com/images" #/thefirstchapter/205.png - set_name = "promos" - num_images = 24 - local_folder = f"images/{set_name}/" - - download_images(base_url, set_name, num_images, local_folder)