COVID-19 Webscraper

In [1]:
# Vaccination clinic data: https://vaccinefinder.nyc.gov/
In [1]:
# import packages
from bs4 import BeautifulSoup
import pandas as pd
import codecs
from urllib.request import urlopen
In [2]:
# file address
f  = codecs.open("NYC COVID-19 and Flu Vaccine Finder.html", 'r','utf-8')

# read in the html file
soup = BeautifulSoup(f, 'html.parser')
## soup
In [3]:
articles = soup.find_all('article', class_ = 'sc-kfzAmx jhOYBm')
print('Total ' + str(len(articles)) + ' vaccine stations as of Jan 11')

Total 1136 vaccine stations as of Jan 11
In [4]:
# collect data from the website
df = pd.DataFrame()
for i in range(len(articles)):
    
    article = articles[i]
    
    name = article.find(class_ ="baseH-sc-1pkt4xd-4 H2-sc-1pkt4xd-6 dtATFP cQDSrG sc-gsTCUz bhdLno")
    if ( name is not None):
        name = name.text
    else:
        name = 'N/A'
    
    address = article.find(class_ ="CalciteP-sc-1pkt4xd-2 sc-bdfBwQ UHspd cIKpxU").text
    
    tp = article.find(class_ ="StyledLabel-sc-19bfcv8-0 eXXBrL").text
    
    tel = article.find(class_ ="CalciteA-sc-1pkt4xd-3 jGotip")
    if (tel is not None):
        tel = tel.text
    else:
        tel = 'N/A'
    

    if (article.find(class_ ="sc-jrAGrp fHSJfL") is not None):
        vacType = article.find(class_ ="sc-jrAGrp fHSJfL").text
    else: 
        vacType = 'N/A'
    
    notes = ''
    for i in range(len(article.find_all(class_ ="sc-jrAGrp gwlDDV"))):
        notes = notes + article.find_all(class_ ="sc-jrAGrp gwlDDV")[i].text + ' , '
    
   
    dfa = {'Name': name, 'Address':address, 'Station_Type':tp, 'Phone_Number':tel,'Vaccine_Type': vacType, 'Notes': notes}
    
    df = df.append(dfa, ignore_index = True)
    
    #print(dfa)
    #ls = ls.append(name)
In [5]:
df = df[['Name','Address','Notes', 'Phone_Number', 'Station_Type', 'Vaccine_Type','Notes']]
df.head(2)
Out[5]:
Name Address Notes Phone_Number Station_Type Vaccine_Type Notes
0 Abyssinian Baptist Church- Pop Up 132 West 138th Street, Manhattan $100 incentive available , Walk-up vaccination... (877) 829-4692 Pop Up - Van Vaccines offered:Pfizer (12+)Johnson & Johnson... $100 incentive available , Walk-up vaccination...
1 AMC Magic Johnson Harlem- Pop Up 2309 Frederick Douglass Blvd, Manhattan $100 incentive available , Walk-up vaccination... (877) 829-4692 Pop Up - Bus Vaccine offered:Pfizer (5-11) $100 incentive available , Walk-up vaccination...
In [6]:
df.to_csv('Vac_stations.csv')
In [ ]:
In [ ]: