Information currently unavailable, check website.
Events currently unavailable, 190 | check website.
sibling where all children are . 37 | if not event_summary: 38 | event_summary = next((t.string for t in event_tags 39 | if t.name == 'p' and t.string and all(c.name == 'b' for c in t.children)), None) 40 | 41 | # Find first
sibling where all children are not . 42 | event_location = next((', '.join(t.stripped_strings) for t in event_tags 43 | if t.name == 'p' and all(c.name != 'b' for c in t.children)), None) 44 | 45 | # Find first sibling and capture entire html. 46 | event_description = next((utils.clean_html(str(t)) for t in event_tags if t.name == 'ul'), None) 47 | 48 | yield CalEventItem( 49 | date=event_date, 50 | summary=event_summary, 51 | location=event_location, 52 | description=event_description 53 | ) 54 | 55 | @staticmethod 56 | def __parse_date(text: str) -> datetime: 57 | return timezone('US/Eastern').localize( 58 | datetime.strptime(text, '%B %d, %I:%M%p').replace(year=datetime.now().year)) 59 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/brooklyn-cb9.py: -------------------------------------------------------------------------------- 1 | import re 2 | from datetime import datetime 3 | 4 | import scrapy 5 | from bs4 import BeautifulSoup 6 | from pytz import timezone 7 | 8 | from cbmap import utils 9 | from cbmap.items import CalEventItem 10 | 11 | 12 | class BrooklynCb9(scrapy.Spider): 13 | name = 'brooklyn-cb9' 14 | title = 'Brooklyn CB9' 15 | 16 | start_urls = [ 17 | 'http://www.communitybrd9bklyn.org/meetings/boardmeetings/', 18 | 'http://www.communitybrd9bklyn.org/meetings/committeemeetings/' 19 | ] 20 | 21 | def parse(self, response): 22 | soup = BeautifulSoup(response.text, 'lxml') 23 | more_info_links = soup.select('.meeting_info a') 24 | for link in more_info_links: 25 | yield scrapy.Request(response.urljoin(link['href']), callback=self.parse_event) 26 | 27 | def parse_event(self, response): 28 | soup = BeautifulSoup(response.text, 'lxml') 29 | 30 | elems = soup.select('.meeting_wrap p') 31 | data = {key.string.rstrip(':').strip(): val.string.strip() 32 | for key, val in (elem.children for elem in elems)} 33 | 34 | date_str = data['Date'] 35 | time_str = data['Time'] 36 | venue = data['Venue'] 37 | address = data['Address'] 38 | 39 | if not date_str or not time_str or time_str == 'CANCELLED': 40 | return 41 | 42 | date_str = utils.strip_date_ords(date_str) 43 | time_str = time_str.replace('.', '') 44 | 45 | event_date = datetime.strptime(date_str, '%B %d, %Y').date() 46 | event_time = self.__parse_time(time_str) 47 | 48 | event_dt = timezone('US/Eastern').localize(datetime.combine(event_date, event_time)) 49 | event_summary = soup.select('.et_main_title')[0].text.strip() 50 | event_description = response.url 51 | event_location = '\n'.join([x for x in (venue, address) if x]) 52 | 53 | yield CalEventItem( 54 | date=event_dt, 55 | summary=event_summary, 56 | description=event_description, 57 | location=event_location 58 | ) 59 | 60 | @staticmethod 61 | def __parse_time(time_str): 62 | try: 63 | return datetime.strptime(time_str, '%I:%M %p').time() 64 | except ValueError: 65 | return datetime.strptime(time_str, '%I:%M%p').time() 66 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/manhattan-cb5.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import scrapy 4 | from bs4 import BeautifulSoup 5 | from pytz import timezone 6 | 7 | from cbmap import utils 8 | from cbmap.items import CalEventItem 9 | 10 | 11 | class ManhattanCb5Spider(scrapy.Spider): 12 | name = 'manhattan-cb5' 13 | title = 'Manhattan CB5' 14 | start_urls = [ 15 | 'https://www.cb5.org/cb5m/calendar' 16 | ] 17 | 18 | def parse(self, response): 19 | soup = BeautifulSoup(response.text, 'lxml') 20 | 21 | depth = response.meta['depth'] or 0 22 | 23 | event_elms = soup.select('#Events div.event') 24 | for elm in event_elms: 25 | event_summary = ' '.join(elm.select_one('h2').stripped_strings) 26 | info = list(elm.select_one('.info').stripped_strings) 27 | event_dt = self.__parse_dt(info[0]) 28 | event_location = '\n'.join(info[1:]) 29 | event_desc = '\n'.join( 30 | (utils.clean_html(str(e)) 31 | for e in elm.select_one('.info').find_next_siblings() 32 | if e.attrs.get('class') != ['up'])) 33 | yield CalEventItem( 34 | date=event_dt, 35 | summary=event_summary, 36 | description=event_desc, 37 | location=event_location 38 | ) 39 | 40 | if depth == 0: 41 | link = soup.select_one('.peernav .next a') 42 | request = scrapy.Request(link['href']) 43 | request.meta['depth'] = depth + 1 44 | yield request 45 | 46 | @staticmethod 47 | def __parse_dt(text): 48 | text = utils.strip_date_ords(text) 49 | dt = datetime.strptime(text, "%A, %B %d, %Y, at %I:%M%p") 50 | return timezone('US/Eastern').localize(dt) 51 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/manhattan-cb7.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class BrooklynCb1Spider(scrapy.Spider): 7 | name = 'manhattan-cb7' 8 | title = 'Manhattan CB7' 9 | start_urls = [ 10 | 'http://www1.nyc.gov/assets/manhattancb7/js/pages/calendar_events.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/queens-cb2.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class QueensCb2Spider(scrapy.Spider): 7 | name = 'queens-cb2' 8 | title = 'Queens CB2' 9 | start_urls = [ 10 | 'http://www.nyc.gov/html/qnscb2/includes/scripts/calendar.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/queens-cb3.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | from bs4 import BeautifulSoup 3 | from icalendar import Calendar 4 | 5 | from cbmap.items import CalEventItem 6 | 7 | 8 | class QueensCb3Spider(scrapy.Spider): 9 | name = 'queens-cb3' 10 | title = 'Queens CB3' 11 | start_urls = [ 12 | 'http://www.cb3qn.nyc.gov/calendar' 13 | ] 14 | 15 | def parse(self, response): 16 | soup = BeautifulSoup(response.text, 'lxml') 17 | 18 | event_links = soup.select('td.feature-cal-inmonth a.feature-cal-inmonth') 19 | for link in event_links: 20 | href = link['href'] 21 | if 'c_eid=' in href: 22 | yield scrapy.Request(response.urljoin(href), callback=self.parse_event) 23 | 24 | depth = response.meta['depth'] or 0 25 | if depth == 0: 26 | next_month_link = soup.find('img', src='/wego/images/next_arrow.gif').parent 27 | request = scrapy.Request(response.urljoin(next_month_link['href'])) 28 | request.meta['depth'] = depth + 1 29 | yield request 30 | 31 | def parse_event(self, response): 32 | soup = BeautifulSoup(response.text, 'lxml') 33 | 34 | ical_link = soup.find('a', text='download calendar file') 35 | 36 | request = scrapy.Request(response.urljoin(ical_link['href']), callback=self.parse_event_ical) 37 | 38 | map_link = soup.find('a', text='Map') 39 | if map_link: 40 | request.meta['location'] = ', '.join( 41 | (x.string.strip() 42 | for x in reversed(list(map_link.previous_siblings)) 43 | if x.string and x.string.strip())) 44 | 45 | yield request 46 | 47 | def parse_event_ical(self, response): 48 | cal = Calendar.from_ical(response.body_as_unicode()) 49 | for vevent in cal.subcomponents: 50 | event_dt = vevent.get('DTSTART').dt 51 | event_summary = vevent.get('SUMMARY') 52 | event_description = vevent.get('DESCRIPTION') 53 | event_location = response.meta.get('location', None) 54 | 55 | if event_summary == 'Alternate Side Parking Rules Suspended': 56 | continue 57 | 58 | yield CalEventItem( 59 | date=event_dt, 60 | summary=event_summary, 61 | description=event_description, 62 | location=event_location 63 | ) 64 | 65 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/queens-cb5.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class QueensCb2Spider(scrapy.Spider): 7 | name = 'queens-cb5' 8 | title = 'Queens CB5' 9 | start_urls = [ 10 | 'http://www1.nyc.gov/assets/queenscb5/js/calendar_events.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/queens-cb6.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | from datetime import datetime 3 | from urllib.parse import urljoin 4 | 5 | import scrapy 6 | from bs4 import BeautifulSoup 7 | 8 | from cbmap.items import CalEventItem 9 | 10 | 11 | class QueensCb6Spider(scrapy.Spider): 12 | name = 'queens-cb6' 13 | title = 'Queens CB6' 14 | start_urls = [ 15 | 'http://www1.nyc.gov/site/queenscb6/news/upcoming-events.page' 16 | ] 17 | 18 | def parse(self, response): 19 | soup = BeautifulSoup(response.text, 'lxml') 20 | title_elms = soup.select('.about-description h2, .about-description h3') 21 | 22 | for elm in title_elms: 23 | siblings = list(itertools.islice( 24 | filter( 25 | lambda x: x.encode().strip() and x.name != 'br', 26 | elm.next_siblings), 27 | 3)) 28 | 29 | # Ensure next 3 elements (ignoring whitespace/s) are text. 30 | if len(siblings) != 3 or not all([x.name is None for x in siblings]): 31 | continue 32 | 33 | summary_text = elm.text 34 | date_text = str(siblings[0]).strip() 35 | time_text = str(siblings[1]).strip() 36 | location_text = str(siblings[2]).strip() 37 | 38 | agenda_elm = elm.find_next_sibling('a') 39 | agenda_href = urljoin(self.start_urls[0], agenda_elm.attrs['href']) 40 | agenda_text = agenda_elm.text 41 | description_text = f' Optional[str]: 73 | if html: 74 | return bleach.clean(html, tags=ALLOWED_TAGS, strip=True) 75 | return None 76 | 77 | 78 | def strip_date_ords(text: str): 79 | return re.sub(r'(?<=\d)(?:th|rd|nd|st)', '', text) -------------------------------------------------------------------------------- /scrapers/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from scrapy.crawler import CrawlerProcess 4 | from scrapy.utils.project import get_project_settings 5 | 6 | 7 | def run_spiders(spiders=None): 8 | setting = get_project_settings() 9 | process = CrawlerProcess(setting) 10 | if not spiders: 11 | spiders = process.spiders.list() 12 | for spider_name in spiders: 13 | process.crawl(spider_name) 14 | process.start() 15 | 16 | 17 | parser = argparse.ArgumentParser(description='CBMap Scrapers!') 18 | parser.add_argument('--all', action='store_true', help='Run all spiders') 19 | parser.add_argument('--spider', action='append', help='Run specified spider') 20 | 21 | args = parser.parse_args() 22 | 23 | run_spiders(None if args.all else args.spider) 24 | -------------------------------------------------------------------------------- /scrapers/runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.6.5 2 | -------------------------------------------------------------------------------- /scrapers/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = cbmap.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = cbmap 12 | -------------------------------------------------------------------------------- /scrapers/server.py: -------------------------------------------------------------------------------- 1 | import os 2 | import threading 3 | import time 4 | 5 | import schedule 6 | 7 | 8 | # https://github.com/dbader/schedule/issues/69 9 | class ScheduleThread(threading.Thread): 10 | def __init__(self, *pargs, **kwargs): 11 | super().__init__(*pargs, daemon=True, name="scheduler", **kwargs) 12 | 13 | def run(self): 14 | while True: 15 | schedule.run_pending() 16 | time.sleep(schedule.idle_seconds()) 17 | 18 | 19 | def scraper_job(): 20 | os.system("python main.py --all") 21 | 22 | 23 | print("cbmap scraper startup!") 24 | 25 | schedule.every(6).hours.do(scraper_job) 26 | 27 | schedule_thread = ScheduleThread() 28 | schedule_thread.start() 29 | 30 | scraper_job() 31 | 32 | schedule_thread.join() 33 | --------------------------------------------------------------------------------