Python Web Scraping Project
By Elad Oz-Cohen in Python
The following code retrives the name and price of an Amazon product an checks it’s prices at fixed intervals untill the product’s price drops below a specified point.
The accumulated data is then stored in a CSV for later use.
Please Feel free to use this code and modify it for your own project.
# Importing libraries
import time
import csv
import datetime
import requests
import os
from bs4 import BeautifulSoup
# Connecting to the Amazon's product webpage.
def check_price():
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
'Accept-Language': 'en-US,en;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
}
page = requests.get(
'https://www.amazon.com/Funny-Data-Systems-Business-Analyst/dp/B07FNW9FGJ/ref=sr_1_3?dchild=1&keywords=data%2Banalyst%2Btshirt&qid=1626655184&sr=8-3&customId=B0752XJYNL&th=1',
headers=headers) # connecting to amazon
# Parsing and cleaning the scraped data and then stores it in a CSV.
soup = BeautifulSoup(page.text, "html.parser") # parsing the HTML result.
title = soup.find(id='productTitle').get_text().strip() # fetching the product's name.
price = soup.find('span', class_="a-offscreen").get_text().strip() # fetching the product's price.
price = float(price.replace('$', '')) # removing the dollar sign.
today = datetime.date.today().strftime("%d-%m-%Y") # getting today's date.
data_itself = [title, price, today]
col_names = ["Title", "Price", "Date"]
with open('Amazon_Price_Data.csv', 'a+', newline='', encoding='UTF8') as f:
writer = csv.writer(f)
f.seek(0, os.SEEK_END) # seek to the end of the file
if f.tell() == 0: # write headers if the file is empty
writer.writerow(col_names)
writer.writerow(data_itself)
return price
# The loop executes every 5 seconds and checks if the price dropped below 15 USD.
while True:
price = check_price()
if price > 15:
print("Price has dropped below 15$. This loop will terminate now.")
break
time.sleep(5)