Pandas only saving first page of scraped data into CSV - pandas

Question

I'm trying to scrape data from the first 5 pages of this site and save it to a CSV. Everything seems to be working fine but only the first page of the site is getting saved to the CSV. I think it may be an indentation issue but I haven't been able to figure it out. Any help would be appreciated, thanks!
from bs4 import BeautifulSoup
import requests
import pandas as pd
import numpy as np
import time
import random
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Cookie': 'pis=3; ut=hB0ZZQzmgNnOh9lBQEu8bTmaWMULGZQUahdMlzzGd1k; SESSION_TOKEN=j-_KSJ8wscb0aeKUf-Ndr5wDcTCFdY3E2qwROIDEt5U; zjs_user_id=null; zjs_anonymous_id=%22hB0ZZQzmgNnOh9lBQEu8bTmaWMULGZQUahdMlzzGd1k%22; _pxvid=ef32ffea-7620-11eb-9542-0242ac12000d; _ga=GA1.2.845026454.1614116940; _gid=GA1.2.991776272.1614116940; _gcl_au=1.1.1819279014.1614116941; OptanonConsent=isIABGlobal=false&datestamp=Tue+Feb+23+2021+16%3A21%3A01+GMT-0600+(Central+Standard+Time)&version=5.9.0&landingPath=NotLandingPage&groups=1%3A1%2C0_172180%3A1%2C0_248632%3A1%2C0_172218%3A1%2C0_172151%3A1%2C0_172362%3A1%2C3%3A1%2C0_172152%3A1%2C0_172351%3A1%2C4%3A1%2C0_172338%3A1%2C0_172360%3A1%2C0_172153%3A1%2C0_172154%3A1%2C0_172343%3A1%2C0_177347%3A1%2C0_172331%3A1%2C0_172155%3A1%2C0_172156%3A1%2C0_248627%3A1%2C0_172157%3A1%2C0_248631%3A1%2C0_172158%3A1%2C0_172357%3A1%2C0_248633%3A1%2C0_172348%3A1%2C0_172159%3A1%2C0_172160%3A1%2C0_172161%3A1%2C0_172162%3A1%2C0_172163%3A1%2C0_172164%3A1%2C0_172165%3A1%2C0_172166%3A1%2C0_172167%3A1%2C0_172168%3A1%2C0_172169%3A1%2C0_172170%3A1%2C0_172171%3A1%2C0_172172%3A1%2C0_172173%3A1%2C0_172174%3A1%2C0_172175%3A1%2C0_172176%3A1%2C0_172177%3A1%2C0_172178%3A1%2C0_172179%3A1%2C0_172181%3A1%2C0_172182%3A1%2C0_172183%3A1%2C0_172184%3A1%2C0_172185%3A1%2C0_172186%3A1%2C0_172187%3A1%2C0_172188%3A1%2C0_172189%3A1%2C0_172190%3A1%2C0_172191%3A1%2C0_172192%3A1%2C0_172193%3A1%2C0_172195%3A1%2C0_172197%3A1%2C0_172198%3A1%2C0_172199%3A1%2C0_172200%3A1%2C0_172201%3A1%2C0_172202%3A1%2C0_172203%3A1%2C0_172204%3A1%2C0_172205%3A1%2C0_172206%3A1%2C0_172207%3A1%2C0_172208%3A1%2C0_172209%3A1%2C0_172210%3A1%2C0_172211%3A1%2C0_172212%3A1%2C0_172213%3A1%2C0_172214%3A1%2C0_172215%3A1%2C0_172216%3A1%2C0_172217%3A1%2C0_172219%3A1%2C0_172220%3A1%2C0_172221%3A1%2C0_172222%3A1%2C0_172223%3A1%2C0_172330%3A1%2C0_172333%3A1%2C0_172334%3A1%2C0_172335%3A1%2C0_172336%3A1%2C0_172337%3A1%2C0_172339%3A1%2C0_172340%3A1%2C0_172341%3A1%2C0_172342%3A1%2C0_172344%3A1%2C0_248628%3A1%2C0_172345%3A1%2C0_172346%3A1%2C0_172349%3A1%2C0_172350%3A1%2C0_172352%3A1%2C0_172353%3A1%2C0_172354%3A1%2C0_172355%3A1%2C0_172356%3A1%2C0_172358%3A1%2C0_172359%3A1%2C0_172361%3A1%2C0_248629%3A1%2C0_248630%3A1%2C0_248634%3A1&AwaitingReconsent=false; _px3=d6e4661fe0f89390bd501cf6d96d7c4ce6da6b629f038f745c417aec166457da:jkvtO/Et7fQoQ9uQjR7cLnpUmnMnTHJjbtYEYxtF8Af3XMaosoyoSH29Qf+5aiOY4Z/BqkATEDsYMrO6hKGNOQ==:1000:v1Auy0PIGkZc2wIJIcWfwOV3SoBz2sZHwNv/67LxTEKseVa/NakBSB7e9s397Ol/RCx/TcpBu3ZoJilwD/sP/3PIkNcxZXjbK+aHVEpfKf37sDvp8iNYyLqZ6QjNsa/0NXHrpVIWto2qgiaU21O2v9R9EgDeaTBEt4MCmMT87V4=',
'Host': 'hotpads.com',
'sec-ch-ua': '"Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87"',
'sec-ch-ua-mobile': '?0',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
}
params = {
'lat: 41.7985 lon: -87.7117 z: 11'
}
pages = np.arange(1, 6)
for page in pages:
page = requests.get('https://hotpads.com/chicago-il/apartments-for-rent' + '?page=' + str(page), headers = headers)
source = page.content
soup = BeautifulSoup(source, 'lxml')
postings = soup.find_all('div', class_="AreaListingsContainer")
time.sleep(random.randint(2,10))
for p in postings:
url = ["https://hotpads.com" + u['href'] for u in p.find_all('a', href=True, text=True)]
address = [a.get_text() for a in p.find_all('h4', class_="styles__ListingCardName-y78yl0-8 jQmZHq")]
price = [p.get_text() for p in p.find_all('div', class_="styles__ListingCardPrice-y78yl0-17 cguwHc")]
beds = [b.get_text() for b in p.find_all('div', class_="styles__ListingCardBedDisplay-y78yl0-7 iPqMa")]
homes = list(zip(url, address, price, beds))
df = pd.DataFrame(homes, columns = ['URL', 'Address', 'Price', 'Beds'])
print(df)
df.to_csv('Chicago_homes.csv')

Answer 1

It looks like you're overwriting the original df in every iteration. There's two solutions to this.
1- Initialize the dataframe before the loop and write the entire frame to the file after the loop has completed.
df = pd.DataFrame()
for page in pages:
#do stuff...
df = pd.concat([df,pd.DataFrame(homes, columns = ['URL', 'Address', 'Price', 'Beds'])])
#put this outside of the loop.
df.to_csv('Chicago_homes.csv')
2- Append the dataframe data to the file within the loop. This is beneficial when executing many loops, and you don't want to hold the large dataframe in memory.
for page in pages:
#do stuff..
if os.path.exists('Chicago_homes.csv'):
df.to_csv('Chicago_homes.csv',mode='a',header=False)
else:
df.to_csv('Chicago_homes.csv')