├── scrapers ├── __init__.py ├── cbmap │ ├── __init__.py │ ├── spiders │ │ ├── __init__.py │ │ ├── bronx-cb5.py │ │ ├── queens-cb2.py │ │ ├── queens-cb5.py │ │ ├── brooklyn-cb1.py │ │ ├── manhattan-cb7.py │ │ ├── bronx-cb2.py │ │ ├── bronx-cb9.py │ │ ├── brooklyn-cb3.py │ │ ├── manhattan-cb5.py │ │ ├── brooklyn-cb9.py │ │ ├── queens-cb3.py │ │ ├── brooklyn-cb6.py │ │ ├── brooklyn-cb2.py │ │ └── queens-cb6.py │ ├── items.py │ ├── serialize.py │ ├── jwtauth.py │ ├── settings.py │ ├── utils.py │ └── pipelines.py ├── runtime.txt ├── Procfile ├── .gitignore ├── scrapy.cfg ├── Pipfile ├── main.py ├── server.py └── Pipfile.lock ├── .dokku-monorepo ├── frontend ├── src │ ├── shared │ │ ├── polyfills.ts │ │ ├── models │ │ │ ├── Size.ts │ │ │ ├── ComponentSizes.ts │ │ │ ├── CalendarEvent.ts │ │ │ ├── Location.ts │ │ │ ├── RootState.ts │ │ │ ├── District.ts │ │ │ ├── DistrictLeadership.ts │ │ │ └── Calendar.ts │ │ ├── types │ │ │ ├── json.d.ts │ │ │ ├── md.d.ts │ │ │ ├── html.d.ts │ │ │ ├── swipeable-views.ts │ │ │ ├── jss-extend.d.ts │ │ │ ├── mapbox-gl.ts │ │ │ ├── promised-location.d.ts │ │ │ ├── react-resize-detector.ts │ │ │ ├── react-jss.d.ts │ │ │ └── mapbox.d.ts │ │ ├── utils │ │ │ └── device.ts │ │ ├── reducers │ │ │ ├── selected-location.ts │ │ │ ├── index.ts │ │ │ └── component-sizes.ts │ │ ├── selectors │ │ │ ├── location-from-route.ts │ │ │ └── district-id-from-route.ts │ │ ├── constants.js │ │ ├── reactGAMiddlewares.js │ │ ├── components │ │ │ └── Html.tsx │ │ ├── icons │ │ │ └── Twitter.tsx │ │ ├── actions │ │ │ └── index.ts │ │ ├── google │ │ │ └── GCalApi.ts │ │ └── data │ │ │ ├── districts-leadership.json │ │ │ └── districts-info.json │ ├── feature │ │ ├── about │ │ │ ├── about.md │ │ │ └── About.tsx │ │ ├── sidebar │ │ │ ├── district_info │ │ │ │ ├── calendar │ │ │ │ │ ├── subscribe-text.html │ │ │ │ │ ├── SubscribeDialog.tsx │ │ │ │ │ ├── EventDialog.tsx │ │ │ │ │ └── CalendarTab.tsx │ │ │ │ ├── LeadershipTab.tsx │ │ │ │ ├── ContactTab.tsx │ │ │ │ └── DistrictInfo.tsx │ │ │ ├── intro │ │ │ │ ├── intro.md │ │ │ │ └── Intro.tsx │ │ │ ├── status │ │ │ │ └── Status.tsx │ │ │ ├── Sidebar.tsx │ │ │ └── search │ │ │ │ └── Search.tsx │ │ └── map │ │ │ └── Map.tsx │ ├── index.css │ ├── app │ │ ├── App.css │ │ ├── App.tsx │ │ └── registerServiceWorker.js │ └── index.tsx ├── public │ ├── google5a5f56e25db68875.html │ ├── favicon.ico │ ├── manifest.json │ └── index.html ├── static.json ├── tsconfig.test.json ├── .gitignore ├── tsconfig.json ├── scripts │ └── get-leadership-data.js ├── config-overrides.js ├── package.json └── tslint.json ├── .gitignore ├── LICENSE └── README.md /scrapers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapers/cbmap/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapers/runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.6.5 2 | -------------------------------------------------------------------------------- /scrapers/Procfile: -------------------------------------------------------------------------------- 1 | worker: python server.py -------------------------------------------------------------------------------- /scrapers/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | output/ 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /.dokku-monorepo: -------------------------------------------------------------------------------- 1 | frontend=frontend 2 | scrapers=scrapers 3 | -------------------------------------------------------------------------------- /frontend/src/shared/polyfills.ts: -------------------------------------------------------------------------------- 1 | import 'core-js/es7/promise'; 2 | -------------------------------------------------------------------------------- /frontend/public/google5a5f56e25db68875.html: -------------------------------------------------------------------------------- 1 | google-site-verification: google5a5f56e25db68875.html -------------------------------------------------------------------------------- /frontend/src/shared/models/Size.ts: -------------------------------------------------------------------------------- 1 | interface Size { 2 | width: number; 3 | height: number; 4 | } 5 | -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codebutler/59boards/HEAD/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/static.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": "build/", 3 | "routes": { 4 | "/**": "index.html" 5 | } 6 | } 7 | 8 | -------------------------------------------------------------------------------- /frontend/src/shared/types/json.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.json' { 2 | const value: any; 3 | export default value; 4 | } 5 | -------------------------------------------------------------------------------- /frontend/src/shared/types/md.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.md' { 2 | const value: string; 3 | export default value; 4 | } 5 | -------------------------------------------------------------------------------- /frontend/src/shared/models/ComponentSizes.ts: -------------------------------------------------------------------------------- 1 | export interface ComponentSizes { 2 | app: Size; 3 | sidebar?: Size; 4 | } 5 | -------------------------------------------------------------------------------- /frontend/src/shared/types/html.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.html' { 2 | const value: string; 3 | export default value; 4 | } 5 | -------------------------------------------------------------------------------- /frontend/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "module": "commonjs" 5 | } 6 | } -------------------------------------------------------------------------------- /frontend/src/shared/types/swipeable-views.ts: -------------------------------------------------------------------------------- 1 | export interface SwipeableViewsChildContext { 2 | slideUpdateHeight: () => void; 3 | } 4 | -------------------------------------------------------------------------------- /frontend/src/feature/about/about.md: -------------------------------------------------------------------------------- 1 | Created By Eric Butler 2 | 3 | * [@codebutler](https://twitter.com/codebutler) 4 | * [GitHub](https://github.com/codebutler/59boards) 5 | -------------------------------------------------------------------------------- /frontend/src/shared/types/jss-extend.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'jss-extend' { 2 | import { JSSPlugin } from 'jss'; 3 | function jssExtend(): JSSPlugin; 4 | export = jssExtend; 5 | } 6 | -------------------------------------------------------------------------------- /frontend/src/shared/models/CalendarEvent.ts: -------------------------------------------------------------------------------- 1 | interface CalendarEvent { 2 | id: string; 3 | date: string; 4 | summary: string; 5 | description: string; 6 | location: string; 7 | } 8 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /frontend/src/index.css: -------------------------------------------------------------------------------- 1 | html { 2 | overflow: hidden; 3 | height: 100%; 4 | } 5 | 6 | body { 7 | height: 100%; 8 | overflow: auto; 9 | margin: 0; 10 | padding: 0; 11 | font-family: sans-serif; 12 | } 13 | -------------------------------------------------------------------------------- /frontend/src/shared/models/Location.ts: -------------------------------------------------------------------------------- 1 | import LngLatLike = mapboxgl.LngLatLike; 2 | import { Coord } from '@turf/helpers'; 3 | 4 | export default interface Location { 5 | place_name?: string; 6 | center: LngLatLike & Coord; 7 | } -------------------------------------------------------------------------------- /frontend/src/shared/types/mapbox-gl.ts: -------------------------------------------------------------------------------- 1 | import 'mapbox-gl'; 2 | import { Feature, GeoJsonProperties, Polygon } from 'geojson'; 3 | 4 | declare module 'mapbox-gl' { 5 | interface MapMouseEvent { 6 | features: Feature[]; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Node build artifacts 2 | node_modules 3 | npm-debug.log 4 | 5 | # Local development 6 | *.env 7 | *.dev 8 | *.swp 9 | .DS_Store 10 | 11 | # Docker 12 | Dockerfile 13 | docker-compose.yml 14 | 15 | # Webstorm 16 | .idea 17 | 18 | frontend/public/scraper-data 19 | -------------------------------------------------------------------------------- /scrapers/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = cbmap.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = cbmap 12 | -------------------------------------------------------------------------------- /frontend/src/shared/models/RootState.ts: -------------------------------------------------------------------------------- 1 | import Location from './Location'; 2 | import { ComponentSizes } from './ComponentSizes'; 3 | import { RouterState } from 'react-router-redux'; 4 | 5 | export interface RootState { 6 | selectedLocation?: Location; 7 | componentSizes: ComponentSizes; 8 | routing?: RouterState; 9 | } 10 | -------------------------------------------------------------------------------- /frontend/src/shared/utils/device.ts: -------------------------------------------------------------------------------- 1 | export function isMobileSafari() { 2 | const userAgent = window.navigator.userAgent; 3 | const isIos = !!userAgent.match(/iP(ad|od|hone)/i); 4 | const isWebKit = !!userAgent.match(/WebKit/i); 5 | const isChrome = !!userAgent.match(/CriOS/i); 6 | return isIos && isWebKit && !isChrome; 7 | } 8 | -------------------------------------------------------------------------------- /frontend/src/shared/reducers/selected-location.ts: -------------------------------------------------------------------------------- 1 | import { ActionType, RootAction } from '../actions'; 2 | 3 | export default (state = null, action: RootAction) => { 4 | switch (action.type) { 5 | case ActionType.SELECT_LOCATION: 6 | return action.location; 7 | default: 8 | return state; 9 | } 10 | }; 11 | -------------------------------------------------------------------------------- /frontend/src/feature/sidebar/district_info/calendar/subscribe-text.html: -------------------------------------------------------------------------------- 1 | To have the latest meetings for this community board automatically appear in your calendar 2 | app, copy and paste the URL below into its subscription feature. See instructions for 3 | Google Calendar and 4 | iCloud. 5 | -------------------------------------------------------------------------------- /frontend/src/shared/models/District.ts: -------------------------------------------------------------------------------- 1 | import { CalendarData } from './Calendar'; 2 | 3 | export default interface District { 4 | id: number; 5 | borough: string; 6 | number: string; 7 | neighborhoods: string; 8 | address?: string; 9 | email?: string; 10 | phone?: string; 11 | twitter?: string; 12 | website?: string; 13 | calendar?: CalendarData; 14 | } 15 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | 6 | # testing 7 | /coverage 8 | 9 | # production 10 | /build 11 | 12 | # misc 13 | .DS_Store 14 | .env.local 15 | .env.development.local 16 | .env.test.local 17 | .env.production.local 18 | 19 | npm-debug.log* 20 | yarn-debug.log* 21 | yarn-error.log* 22 | 23 | .idea 24 | 25 | -------------------------------------------------------------------------------- /frontend/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "59Boards", 3 | "name": "59Boards: NYC Community Board Map", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": "./index.html", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /frontend/src/shared/reducers/index.ts: -------------------------------------------------------------------------------- 1 | import { combineReducers } from 'redux'; 2 | import selectedLocation from './selected-location'; 3 | import componentSizes from './component-sizes'; 4 | import { routerReducer } from 'react-router-redux'; 5 | 6 | const allReducers = combineReducers({ 7 | selectedLocation, 8 | componentSizes, 9 | routing: routerReducer 10 | }); 11 | 12 | export default allReducers; 13 | -------------------------------------------------------------------------------- /frontend/src/shared/types/promised-location.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'promised-location' { 2 | interface PromisedLocationOptions { 3 | enableHighAccuracy: boolean; 4 | timeout: number; 5 | maximumAge: number; 6 | } 7 | 8 | class PromisedLocation extends Promise { 9 | constructor(options: PromisedLocationOptions); 10 | } 11 | 12 | export = PromisedLocation; 13 | } 14 | -------------------------------------------------------------------------------- /scrapers/cbmap/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | from scrapy import Field 10 | 11 | 12 | class CalEventItem(scrapy.Item): 13 | id = Field() 14 | date = Field() 15 | summary = Field() 16 | location = Field() 17 | description = Field() 18 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/bronx-cb5.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class BrooklynCb1Spider(scrapy.Spider): 7 | name = 'bronx-cb5' 8 | title = 'Bronx CB5' 9 | start_urls = [ 10 | 'http://www1.nyc.gov/assets/bronxcb5/js/calendar_events.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/queens-cb2.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class QueensCb2Spider(scrapy.Spider): 7 | name = 'queens-cb2' 8 | title = 'Queens CB2' 9 | start_urls = [ 10 | 'http://www.nyc.gov/html/qnscb2/includes/scripts/calendar.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/queens-cb5.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class QueensCb2Spider(scrapy.Spider): 7 | name = 'queens-cb5' 8 | title = 'Queens CB5' 9 | start_urls = [ 10 | 'http://www1.nyc.gov/assets/queenscb5/js/calendar_events.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/brooklyn-cb1.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class BrooklynCb1Spider(scrapy.Spider): 7 | name = 'brooklyn-cb1' 8 | title = 'Queens CB1' 9 | start_urls = [ 10 | 'http://www.nyc.gov/html/bkncb1/includes/scripts/calendar.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /frontend/src/shared/selectors/location-from-route.ts: -------------------------------------------------------------------------------- 1 | import { RootState } from '../models/RootState'; 2 | 3 | const locationFromRoute = (state: RootState): string | undefined => { 4 | if (state.routing && state.routing.location) { 5 | const params = new URLSearchParams(state.routing.location.search); 6 | return params.get('q') || undefined; 7 | } 8 | return undefined; 9 | }; 10 | 11 | export default locationFromRoute; 12 | -------------------------------------------------------------------------------- /scrapers/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | 3 | url = "https://pypi.python.org/simple" 4 | verify_ssl = true 5 | name = "pypi" 6 | 7 | 8 | [packages] 9 | 10 | twisted = { git = "https://github.com/twisted/twisted.git", ref="9384-remove-async-param" } 11 | bleach = "*" 12 | scrapy = "*" 13 | "bs4" = "*" 14 | parsedatetime = "*" 15 | icalendar = "*" 16 | pytz = "*" 17 | schedule = "*" 18 | requests = "*" 19 | raven = "*" 20 | 21 | 22 | [dev-packages] 23 | 24 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/manhattan-cb7.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | 3 | from cbmap.utils import parse_calendarjs 4 | 5 | 6 | class BrooklynCb1Spider(scrapy.Spider): 7 | name = 'manhattan-cb7' 8 | title = 'Manhattan CB7' 9 | start_urls = [ 10 | 'http://www1.nyc.gov/assets/manhattancb7/js/pages/calendar_events.js' 11 | ] 12 | 13 | def parse(self, response): 14 | for item in parse_calendarjs(response): 15 | yield item 16 | -------------------------------------------------------------------------------- /frontend/src/shared/types/react-resize-detector.ts: -------------------------------------------------------------------------------- 1 | declare module 'react-resize-detector' { 2 | 3 | interface ReactResizeDetectorProps extends React.Props { 4 | handleHeight?: boolean; 5 | handleWidth?: boolean; 6 | onResize: (width: number, height: number) => void; 7 | } 8 | 9 | class ReactResizeDetector extends React.Component { } 10 | 11 | export = ReactResizeDetector; 12 | } 13 | -------------------------------------------------------------------------------- /frontend/src/shared/types/react-jss.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'react-jss/lib/JssProvider' { 2 | import * as React from 'react'; 3 | import 'react-jss/lib/JssProvider'; 4 | import { GenerateClassName } from 'jss'; 5 | 6 | interface JssProviderProps { 7 | jss: object; 8 | generateClassName: GenerateClassName; 9 | } 10 | 11 | const JssProvider: React.ComponentType; 12 | 13 | export default JssProvider; 14 | } 15 | -------------------------------------------------------------------------------- /scrapers/cbmap/serialize.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import pytz 4 | 5 | DATE_FORMAT = "%Y-%m-%d" 6 | TIME_FORMAT = "%H:%M:%SZ" 7 | 8 | 9 | # https://github.com/scrapy/scrapy/issues/2087 10 | def json_default(obj): 11 | if isinstance(obj, datetime.datetime): 12 | return obj.astimezone(pytz.utc).strftime("%sT%s" % (DATE_FORMAT, TIME_FORMAT)) 13 | elif isinstance(obj, datetime.date): 14 | return obj.strftime("%s" % DATE_FORMAT) 15 | else: 16 | raise TypeError(f"Type {type(obj)} not serializable") 17 | -------------------------------------------------------------------------------- /frontend/src/shared/models/DistrictLeadership.ts: -------------------------------------------------------------------------------- 1 | export default interface DistrictLeadership { 2 | communityBoard: string; 3 | chair: string; 4 | districtManager: string; 5 | boardMeetingCabinetMeeting: string; 6 | address1: string; 7 | address2: string; 8 | phoneNumber: string; 9 | postcode: string; 10 | borough: string; 11 | latitude: string; 12 | longitude: string; 13 | councilDistrict: string; 14 | censusTract: string; 15 | bin: string; 16 | bbl: string; 17 | nta: string; 18 | location1: string; 19 | } 20 | -------------------------------------------------------------------------------- /frontend/src/shared/reducers/component-sizes.ts: -------------------------------------------------------------------------------- 1 | import { ActionType, RootAction } from '../actions'; 2 | import { ComponentSizes } from '../models/ComponentSizes'; 3 | 4 | const DEFAULT_STATE: ComponentSizes = { 5 | app: { width: 0, height: 0} 6 | }; 7 | 8 | export default (state: ComponentSizes = DEFAULT_STATE, action: RootAction) => { 9 | switch (action.type) { 10 | case ActionType.COMPONENT_RESIZED: 11 | const newState = { ...state }; 12 | newState[action.id] = action.size; 13 | return newState; 14 | default: 15 | return state; 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /frontend/src/shared/constants.js: -------------------------------------------------------------------------------- 1 | exports = module.exports = { 2 | 'MAPBOX_TOKEN': 'pk.eyJ1IjoiY29kZWJ1dGxlciIsImEiOiJjajhvdnI5NHkwOXppMndubnoyYjdhZHN2In0.evH8Wvs-5rlT29mOaWEoCA', 3 | 4 | /* 5 | const dummyBoroCDs = this.map.querySourceFeatures("districts") 6 | .map((feature) => feature.properties.BoroCD) 7 | .filter((boroCD) => !DISTRICTS[boroCD] ); 8 | */ 9 | 'DUMMY_BORO_IDS': [481, 484, 482, 227, 355, 483, 480, 164, 226, 228, 356, 481, 484, 482, 227, 355, 483, 480, 164, 226, 595, 228, 356], 10 | 'NYC_BOUNDING_BOX': [[-74.2555928790469, 40.49612360054], [-73.7000104159344, 40.915541075395]], 11 | 'NYC_CENTER': { latitude: 40.7128, longitude: -74.0060 }, 12 | }; -------------------------------------------------------------------------------- /frontend/src/shared/reactGAMiddlewares.js: -------------------------------------------------------------------------------- 1 | import ReactGA from 'react-ga' 2 | 3 | const options = {}; 4 | 5 | const trackPage = (page) => { 6 | ReactGA.set({ 7 | page, 8 | ...options 9 | }); 10 | ReactGA.pageview(page) 11 | }; 12 | 13 | let currentPage = ''; 14 | 15 | export const googleAnalytics = store => next => action => { 16 | if (action.type === '@@router/LOCATION_CHANGE') { 17 | const nextPage = `${action.payload.pathname}${action.payload.search}`; 18 | 19 | if (currentPage !== nextPage) { 20 | currentPage = nextPage; 21 | trackPage(nextPage) 22 | } 23 | } 24 | 25 | return next(action) 26 | }; 27 | -------------------------------------------------------------------------------- /scrapers/cbmap/jwtauth.py: -------------------------------------------------------------------------------- 1 | from scrapy import signals 2 | 3 | 4 | class JWTAuthMiddleware(object): 5 | 6 | auth = None 7 | 8 | @classmethod 9 | def from_crawler(cls, crawler): 10 | o = cls() 11 | crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) 12 | return o 13 | 14 | def spider_opened(self, spider): 15 | jwt = getattr(spider, 'jwt', '') 16 | if jwt: 17 | self.auth = f'Bearer {jwt}' 18 | 19 | def process_request(self, request, spider): 20 | auth = getattr(self, 'auth', None) 21 | if auth and b'Authorization' not in request.headers: 22 | request.headers[b'Authorization'] = auth 23 | -------------------------------------------------------------------------------- /frontend/src/app/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | position: absolute; 3 | overflow: hidden; 4 | top: 0; 5 | left: 0; 6 | width: 100%; 7 | height: 100%; 8 | } 9 | 10 | .App-logo { 11 | animation: App-logo-spin infinite 20s linear; 12 | height: 80px; 13 | } 14 | 15 | .App-header { 16 | background-color: #222; 17 | height: 150px; 18 | padding: 20px; 19 | color: white; 20 | } 21 | 22 | .App-title { 23 | font-size: 1.5em; 24 | } 25 | 26 | .App-intro { 27 | font-size: large; 28 | } 29 | 30 | .mapboxgl-map { 31 | position: absolute !important; 32 | top: 0 !important; 33 | right: 0 !important; 34 | left: 0 !important; 35 | bottom: 0 !important; 36 | } 37 | 38 | .mapboxgl-popup-content { 39 | pointer-events: none !important; 40 | } 41 | 42 | -------------------------------------------------------------------------------- /frontend/src/shared/selectors/district-id-from-route.ts: -------------------------------------------------------------------------------- 1 | import { matchPath } from 'react-router'; 2 | import { RootState } from '../models/RootState'; 3 | 4 | interface DistrictIdPathParams { 5 | selectedDistrictId?: string; 6 | } 7 | 8 | const districtIdFromRoute = (state: RootState): number | undefined => { 9 | if (state.routing && state.routing.location) { 10 | const match = matchPath(state.routing.location.pathname, { 11 | path: '/districts/:selectedDistrictId', 12 | exact: true 13 | }); 14 | if (match) { 15 | return parseInt(match.params.selectedDistrictId!, 10); 16 | } 17 | } 18 | return undefined; 19 | }; 20 | 21 | export default districtIdFromRoute; 22 | -------------------------------------------------------------------------------- /scrapers/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from scrapy.crawler import CrawlerProcess 4 | from scrapy.utils.project import get_project_settings 5 | 6 | 7 | def run_spiders(spiders=None): 8 | setting = get_project_settings() 9 | process = CrawlerProcess(setting) 10 | if not spiders: 11 | spiders = process.spiders.list() 12 | for spider_name in spiders: 13 | process.crawl(spider_name) 14 | process.start() 15 | 16 | 17 | parser = argparse.ArgumentParser(description='CBMap Scrapers!') 18 | parser.add_argument('--all', action='store_true', help='Run all spiders') 19 | parser.add_argument('--spider', action='append', help='Run specified spider') 20 | 21 | args = parser.parse_args() 22 | 23 | run_spiders(None if args.all else args.spider) 24 | -------------------------------------------------------------------------------- /scrapers/server.py: -------------------------------------------------------------------------------- 1 | import os 2 | import threading 3 | import time 4 | 5 | import schedule 6 | 7 | 8 | # https://github.com/dbader/schedule/issues/69 9 | class ScheduleThread(threading.Thread): 10 | def __init__(self, *pargs, **kwargs): 11 | super().__init__(*pargs, daemon=True, name="scheduler", **kwargs) 12 | 13 | def run(self): 14 | while True: 15 | schedule.run_pending() 16 | time.sleep(schedule.idle_seconds()) 17 | 18 | 19 | def scraper_job(): 20 | os.system("python main.py --all") 21 | 22 | 23 | print("cbmap scraper startup!") 24 | 25 | schedule.every(6).hours.do(scraper_job) 26 | 27 | schedule_thread = ScheduleThread() 28 | schedule_thread.start() 29 | 30 | scraper_job() 31 | 32 | schedule_thread.join() 33 | -------------------------------------------------------------------------------- /scrapers/cbmap/settings.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from raven.conf import setup_logging 5 | from raven.handlers.logging import SentryHandler 6 | 7 | BOT_NAME = 'cbmap' 8 | 9 | SPIDER_MODULES = ['cbmap.spiders'] 10 | NEWSPIDER_MODULE = 'cbmap.spiders' 11 | 12 | ROBOTSTXT_OBEY = True 13 | 14 | DOWNLOADER_MIDDLEWARES = { 15 | 'cbmap.jwtauth.JWTAuthMiddleware': 100 16 | } 17 | 18 | ITEM_PIPELINES = { 19 | 'cbmap.pipelines.IdPipeline': 300, 20 | 'cbmap.pipelines.MissingSummaryPipeline': 350, 21 | 'cbmap.pipelines.JsonWriterPipeline': 400, 22 | 'cbmap.pipelines.ICalWriterPipeline': 500, 23 | 'cbmap.pipelines.CounterPipeline': 500, 24 | } 25 | 26 | SENTRY_DSN = os.getenv('SENTRY_DSN', None) 27 | if SENTRY_DSN: 28 | handler = SentryHandler(SENTRY_DSN, level=logging.ERROR) 29 | setup_logging(handler) 30 | -------------------------------------------------------------------------------- /frontend/src/feature/sidebar/intro/intro.md: -------------------------------------------------------------------------------- 1 | New York City's government includes a system of 59 community board districts that offer a way for the public to get involved with local politics. Many New Yorkers don't know they exist, and the process to get involved can seem overwhelming. 2 | 3 | Most boards don't provide an easy way to get notified of upcoming meetings, making participation difficult. 4 | 59Boards [scrapes](https://en.wikipedia.org/wiki/Web_scraping) event information from board websites and provides a feed for your calendar app's subscription feature. 5 | 6 | Enter your address below or select a district on the map to get started! 7 | 8 | * [About Community Boards (nyc.gov)](http://www.nyc.gov/html/cau/html/cb/about.shtml) 9 | * [How to join your community board (Curbed)](https://ny.curbed.com/2017/3/15/14918194/nyc-community-board-member-get-involved) 10 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "outDir": "build/dist", 5 | "module": "esnext", 6 | "target": "es5", 7 | "lib": ["es7", "dom"], 8 | "sourceMap": true, 9 | "allowJs": true, 10 | "jsx": "react", 11 | "moduleResolution": "node", 12 | "rootDir": "src", 13 | "forceConsistentCasingInFileNames": true, 14 | "noImplicitReturns": true, 15 | "noImplicitThis": true, 16 | "noImplicitAny": true, 17 | "strictNullChecks": true, 18 | "suppressImplicitAnyIndexErrors": true, 19 | "noUnusedLocals": true, 20 | "allowSyntheticDefaultImports": true, 21 | "experimentalDecorators": true 22 | }, 23 | "exclude": [ 24 | "node_modules", 25 | "build", 26 | "scripts", 27 | "acceptance-tests", 28 | "webpack", 29 | "jest", 30 | "src/setupTests.ts" 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /frontend/src/shared/types/mapbox.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace mapbox { 2 | interface LatLng { 3 | latitude: number; 4 | longitude: number; 5 | } 6 | 7 | interface GeocodeForwardOptions { 8 | autocomplete: boolean; 9 | bbox: number[]; 10 | proximity: LatLng; 11 | } 12 | 13 | // https://github.com/mapbox/carmen/blob/master/carmen-geojson.md 14 | interface CarmenLocation { 15 | place_name: string; 16 | place_type: string[]; 17 | } 18 | } 19 | 20 | declare module 'mapbox' { 21 | import { ResponsePromise } from "rest"; 22 | import GeocodeForwardOptions = mapbox.GeocodeForwardOptions; 23 | 24 | class MapboxClient { 25 | constructor(token: String); 26 | geocodeForward(query: String, options: GeocodeForwardOptions): ResponsePromise; 27 | } 28 | 29 | export = MapboxClient; 30 | } 31 | -------------------------------------------------------------------------------- /frontend/src/shared/components/Html.tsx: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import { Theme, WithStyles } from 'material-ui'; 3 | import withStyles from 'material-ui/styles/withStyles'; 4 | 5 | interface Props { 6 | html: string; 7 | } 8 | 9 | type ClassKey = 10 | | 'root'; 11 | 12 | type PropsWithStyles = Props & WithStyles; 13 | 14 | class Html extends Component { 15 | render() { 16 | const { classes } = this.props; 17 | return ( 18 | 22 | ); 23 | } 24 | } 25 | 26 | const styles = (theme: Theme) => ({ 27 | root: { 28 | '& ul': { 29 | paddingLeft: theme.spacing.unit * 2, 30 | } 31 | } 32 | }); 33 | 34 | export default withStyles(styles)(Html); 35 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/bronx-cb2.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | from icalendar import Calendar 3 | from pytz import timezone 4 | 5 | from cbmap.items import CalEventItem 6 | 7 | 8 | class BronxCb2Spider(scrapy.Spider): 9 | name = 'bronx-cb2' 10 | title = 'Bronx CB2' 11 | 12 | start_urls = [ 13 | 'http://bxcb2.org/events/?ical=1&tribe_display=month' 14 | ] 15 | 16 | def parse(self, response): 17 | cal = Calendar.from_ical(response.body_as_unicode()) 18 | for vevent in cal.subcomponents: 19 | event_dt = timezone('US/Eastern').localize(vevent.get('DTSTART').dt) 20 | event_id = vevent.get('UID') 21 | event_summary = vevent.get('SUMMARY') 22 | event_location = vevent.get('LOCATION') 23 | event_url = vevent.get('URL') 24 | yield CalEventItem( 25 | id=event_id, 26 | date=event_dt, 27 | summary=event_summary, 28 | description=event_url, 29 | location=event_location 30 | ) 31 | 32 | -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 59Boards - NYC Community Board Map 17 | 18 | 19 | 22 |
23 | 24 | 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018-present 59Boards Contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/bronx-cb9.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import scrapy 4 | from bs4 import BeautifulSoup 5 | 6 | from cbmap.items import CalEventItem 7 | 8 | 9 | class BronxCb9Spider(scrapy.Spider): 10 | name = 'bronx-cb9' 11 | title = 'Bronx CB9' 12 | start_urls = [ 13 | 'http://www1.nyc.gov/site/bronxcb9/calendar/calendar.page' 14 | ] 15 | 16 | def parse(self, response): 17 | soup = BeautifulSoup(response.text, 'lxml') 18 | list_items = soup.select('.about-description li') 19 | for li in list_items: 20 | title_elm = li.select_one('b') 21 | 22 | summary = ''.join(title_elm.stripped_strings) 23 | datetime_text = ''.join(title_elm.find_previous_siblings(text=True)).strip() 24 | location = ''.join(title_elm.find_next_siblings(text=True)).strip() 25 | 26 | if '-' in datetime_text: 27 | datetime_text = datetime_text.split('-')[0].strip() 28 | 29 | event_dt = datetime.strptime(datetime_text, '%A, %B %d, %I%p').replace(year=datetime.now().year) 30 | 31 | yield CalEventItem( 32 | date=event_dt, 33 | summary=summary, 34 | description=None, 35 | location=location 36 | ) 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 59Boards: NYC Community Board Map & Meetings Calendar Scraper 2 | 3 | New York City's government includes a system of 59 community board districts that offer a way for the public to get involved with local politics. Many New Yorkers don't know they exist, and the process to get involved can seem overwhelming. 4 | 5 | Most boards don't provide an easy way to get notified of upcoming meetings, making participation difficult. 59Boards [scrapes](https://en.wikipedia.org/wiki/Web_scraping) event information from board websites and provides a feed for your calendar app's subscription feature. 6 | 7 | ## Chat 8 | Join us in `#app-59boards` on the [Beta NYC Slack](http://slack.beta.nyc/). 9 | 10 | ## frontend 11 | 12 | ``` 13 | $ cd frontend 14 | $ npm install 15 | $ npm start 16 | ``` 17 | 18 | ## scrapers 19 | ``` 20 | $ ln -s ../../scrapers/output frontend/public/scraper-data 21 | $ cd scrapers 22 | $ pip install virtualenv 23 | $ virtualenv venv 24 | $ . venv/bin/activate 25 | $ pip install pipenv 26 | $ pipenv install 27 | $ python main.py --all 28 | ``` 29 | 30 | You'll also need [tabula-server](https://github.com/codebutler/tabula-server/) running on localhost:4000 for the PDF scrapers to work. 31 | 32 | ## License 33 | 34 | This project is licensed under the terms of the [MIT license](/LICENSE). 35 | -------------------------------------------------------------------------------- /frontend/scripts/get-leadership-data.js: -------------------------------------------------------------------------------- 1 | const request = require('request'); 2 | const csv = require('csv-parser'); 3 | const t2map = require('through2-map'); 4 | const _ = require('lodash'); 5 | const fs = require('fs'); 6 | const path = require('path'); 7 | const JSONStream = require('JSONStream'); 8 | 9 | const OUTPUT_DIR = path.join(path.dirname(__filename), '..', 'src', 'shared', 'data'); 10 | const DISTRICT_INFO = require(path.join(OUTPUT_DIR, 'districts-info.json')); 11 | 12 | const findDistrict = (borough, number) => _.find(DISTRICT_INFO, district => 13 | district.borough.toLowerCase() === borough.toLowerCase() 14 | && parseInt(district.number) === parseInt(number)); 15 | 16 | // Community Board Leadership (Manhattan) 17 | // https://data.cityofnewyork.us/City-Government/Community-Board-Leadership/3gkd-ddzn 18 | 19 | request.get('https://data.cityofnewyork.us/api/views/3gkd-ddzn/rows.csv?accessType=DOWNLOAD') 20 | .pipe(csv()) 21 | .pipe(t2map.obj(item => [ 22 | findDistrict( 23 | item['Borough'] || 'Manhattan', 24 | item['Community Board'] 25 | )['id'].toString(), 26 | _(item) 27 | .mapKeys((value, key) => _.camelCase(key)) 28 | .mapValues((value) => value.trim())])) 29 | .pipe(JSONStream.stringifyObject('{\n', ',\n', '\n}\n', ' ')) 30 | .pipe(fs.createWriteStream(path.join(OUTPUT_DIR, 'districts-leadership.json'))); 31 | -------------------------------------------------------------------------------- /frontend/src/shared/icons/Twitter.tsx: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import SvgIcon from 'material-ui/SvgIcon'; 3 | 4 | /* tslint:disable */ 5 | 6 | export default class Twitter extends Component { 7 | 8 | render() { 9 | return ( 10 | 11 | 12 | 13 | ); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import { Provider } from 'react-redux'; 4 | import { applyMiddleware, createStore } from 'redux'; 5 | import { createLogger } from 'redux-logger'; 6 | import thunk from 'redux-thunk'; 7 | import 'typeface-roboto'; 8 | import App from './app/App'; 9 | import './index.css'; 10 | import reducer from './shared/reducers'; 11 | import registerServiceWorker from './app/registerServiceWorker'; 12 | import { ConnectedRouter, routerMiddleware } from 'react-router-redux'; 13 | import { composeWithDevTools } from 'redux-devtools-extension'; 14 | import createBrowserHistory from 'history/createBrowserHistory'; 15 | import './shared/polyfills'; 16 | import { googleAnalytics } from './shared/reactGAMiddlewares'; 17 | import Raven from 'raven-js'; 18 | 19 | Raven.config('https://e2202585cda74a63ae066e88f237596d@sentry.io/302432') 20 | .install(); 21 | 22 | const LOGGING = false; 23 | const CACHING = false; 24 | 25 | const history = createBrowserHistory(); 26 | 27 | const middlewares = []; 28 | middlewares.push(thunk); 29 | middlewares.push(routerMiddleware(history)); 30 | middlewares.push(googleAnalytics); 31 | if (LOGGING) { 32 | middlewares.push(createLogger({ 33 | diff: true 34 | })); 35 | } 36 | 37 | const store = createStore( 38 | reducer, composeWithDevTools( 39 | applyMiddleware(...middlewares) 40 | )); 41 | 42 | ReactDOM.render( 43 | 44 | 45 | 46 | 47 | , 48 | document.getElementById('root') 49 | ); 50 | 51 | if (CACHING) { 52 | registerServiceWorker(); 53 | } 54 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/brooklyn-cb3.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import urllib.parse 4 | from datetime import datetime 5 | 6 | import scrapy 7 | 8 | from cbmap.items import CalEventItem 9 | 10 | DEV_TABULA_URL = 'http://localhost:4000/scrape' 11 | DEV_TABULA_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ0YWJ1bGFzZXJ2ZXIiLCJpYX' \ 12 | 'QiOjE1MjEwNTkyNTUsImV4cCI6MTU1MjU5NTI2NiwiYXVkIjoidGFidWxhc2VydmVyIiwic' \ 13 | '3ViIjoidGFidWxhc2VydmVyIn0.d_3_pDKA4ihwriTkueA_u5CDuntndY1Vqly37jVJXcU' 14 | 15 | 16 | class BrooklynCb3Spider(scrapy.Spider): 17 | name = 'brooklyn-cb3' 18 | title = 'Brooklyn CB3' 19 | 20 | tabula_url = os.environ.get('TABULA_URL', DEV_TABULA_URL) 21 | jwt = os.environ.get('TABULA_TOKEN', DEV_TABULA_TOKEN) 22 | 23 | pdf_url = 'http://www1.nyc.gov/assets/brooklyncb3/downloads/pdf/committee_meetings_2018_calendar.pdf' 24 | 25 | start_urls = [ 26 | f'{tabula_url}/?{urllib.parse.urlencode({"url":pdf_url})}' 27 | ] 28 | 29 | def parse(self, response): 30 | pdf = json.loads(response.body_as_unicode()) 31 | 32 | data = pdf['pages'][0]['tables'][0]['data'] 33 | for event in data: 34 | committee = event['COMMITTEE'] 35 | chair = event['CHAIR/VICE-CHAIR/CO-CHAIR'] 36 | time = event['TIME'] 37 | date = event['DATE'] 38 | 39 | event_time = datetime.strptime(time, '%I:%M %p').time() 40 | event_date = datetime.strptime(date, '%d-%b-%y').date() 41 | event_dt = datetime.combine(event_date, event_time) 42 | 43 | yield CalEventItem( 44 | date=event_dt, 45 | summary=committee, 46 | description=chair, 47 | location=None 48 | ) 49 | 50 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/manhattan-cb5.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import scrapy 4 | from bs4 import BeautifulSoup 5 | from pytz import timezone 6 | 7 | from cbmap import utils 8 | from cbmap.items import CalEventItem 9 | 10 | 11 | class ManhattanCb5Spider(scrapy.Spider): 12 | name = 'manhattan-cb5' 13 | title = 'Manhattan CB5' 14 | start_urls = [ 15 | 'https://www.cb5.org/cb5m/calendar' 16 | ] 17 | 18 | def parse(self, response): 19 | soup = BeautifulSoup(response.text, 'lxml') 20 | 21 | depth = response.meta['depth'] or 0 22 | 23 | event_elms = soup.select('#Events div.event') 24 | for elm in event_elms: 25 | event_summary = ' '.join(elm.select_one('h2').stripped_strings) 26 | info = list(elm.select_one('.info').stripped_strings) 27 | event_dt = self.__parse_dt(info[0]) 28 | event_location = '\n'.join(info[1:]) 29 | event_desc = '\n'.join( 30 | (utils.clean_html(str(e)) 31 | for e in elm.select_one('.info').find_next_siblings() 32 | if e.attrs.get('class') != ['up'])) 33 | yield CalEventItem( 34 | date=event_dt, 35 | summary=event_summary, 36 | description=event_desc, 37 | location=event_location 38 | ) 39 | 40 | if depth == 0: 41 | link = soup.select_one('.peernav .next a') 42 | request = scrapy.Request(link['href']) 43 | request.meta['depth'] = depth + 1 44 | yield request 45 | 46 | @staticmethod 47 | def __parse_dt(text): 48 | text = utils.strip_date_ords(text) 49 | dt = datetime.strptime(text, "%A, %B %d, %Y, at %I:%M%p") 50 | return timezone('US/Eastern').localize(dt) 51 | -------------------------------------------------------------------------------- /frontend/config-overrides.js: -------------------------------------------------------------------------------- 1 | const paths = require('react-scripts-ts/config/paths'); 2 | 3 | const path = require("path"); 4 | const ruleChildren = (loader) => loader.use || loader.oneOf || Array.isArray(loader.loader) && loader.loader || [] 5 | 6 | const findIndexAndRules = (rulesSource, ruleMatcher) => { 7 | let result = undefined 8 | const rules = Array.isArray(rulesSource) ? rulesSource : ruleChildren(rulesSource) 9 | rules.some((rule, index) => result = ruleMatcher(rule) ? {index, rules} : findIndexAndRules(ruleChildren(rule), ruleMatcher)) 10 | return result 11 | } 12 | 13 | const createLoaderMatcher = (loader) => (rule) => rule.loader && rule.loader.indexOf(`${path.sep}${loader}${path.sep}`) !== -1 14 | const fileLoaderMatcher = createLoaderMatcher('file-loader') 15 | 16 | const addBeforeRule = (rulesSource, ruleMatcher, value) => { 17 | const {index, rules} = findIndexAndRules(rulesSource, ruleMatcher) 18 | rules.splice(index, 0, value) 19 | } 20 | 21 | module.exports = function override(config) { 22 | config.module.rules.push({ 23 | test: /\.(js|jsx)$/, 24 | include: paths.appSrc, 25 | loader: require.resolve('babel-loader'), 26 | options: { 27 | babelrc: false, 28 | presets: [require.resolve('babel-preset-react-app')], 29 | cacheDirectory: true, 30 | }, 31 | }); 32 | addBeforeRule(config.module.rules, fileLoaderMatcher, { 33 | test: /\.html$/, 34 | use: [ 35 | { loader: require.resolve('html-loader') } 36 | ] 37 | }); 38 | addBeforeRule(config.module.rules, fileLoaderMatcher, { 39 | test: /\.md$/, 40 | use: [ 41 | { loader: require.resolve('html-loader') }, 42 | { loader: require.resolve('markdown-loader') } 43 | ] 44 | }); 45 | return config; 46 | }; 47 | -------------------------------------------------------------------------------- /frontend/src/feature/sidebar/district_info/LeadershipTab.tsx: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import District from '../../../shared/models/District'; 3 | import DISTRICT_LEADERSHIP from '../../../shared/data/districts-leadership.json'; 4 | import DistrictLeadership from '../../../shared/models/DistrictLeadership'; 5 | import { List, ListItem, ListItemText, WithStyles } from 'material-ui'; 6 | import withStyles from 'material-ui/styles/withStyles'; 7 | import { Theme } from 'material-ui/styles'; 8 | 9 | interface Props { 10 | district: District; 11 | } 12 | 13 | type ClassKey = 14 | | 'emptyListContainer'; 15 | 16 | type PropsWithStyles = Props & WithStyles; 17 | 18 | class LeadershipTab extends Component { 19 | render() { 20 | const { classes } = this.props; 21 | const leadership = DISTRICT_LEADERSHIP[this.props.district.id] as DistrictLeadership; 22 | if (!leadership) { 23 | return ( 24 |
25 |

Information currently unavailable, check website.

26 |
27 | ); 28 | } 29 | return ( 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | ); 39 | } 40 | } 41 | 42 | const styles = (theme: Theme) => ( 43 | { 44 | emptyListContainer: { 45 | textAlign: 'center' as 'center', 46 | paddingTop: theme.spacing.unit * 3, 47 | paddingBottom: theme.spacing.unit, 48 | } 49 | } 50 | ); 51 | 52 | export default withStyles(styles)(LeadershipTab); 53 | -------------------------------------------------------------------------------- /frontend/src/feature/about/About.tsx: -------------------------------------------------------------------------------- 1 | import { Component, default as React } from 'react'; 2 | import Dialog, { DialogActions, DialogContent, DialogContentText, DialogTitle } from 'material-ui/Dialog'; 3 | import Button from 'material-ui/Button'; 4 | import { connect } from 'react-redux'; 5 | import { RootAction } from '../../shared/actions'; 6 | import { Dispatch } from 'redux'; 7 | import { push } from 'react-router-redux'; 8 | import { bind } from 'lodash-decorators/bind'; 9 | import Html from '../../shared/components/Html'; 10 | import AboutHtml from './about.md'; 11 | 12 | interface DispatchProps { 13 | onDialogExited: () => void; 14 | } 15 | 16 | type Props = DispatchProps; 17 | 18 | interface State { 19 | isOpen: boolean; 20 | } 21 | 22 | class About extends Component { 23 | 24 | constructor(props: Props) { 25 | super(props); 26 | this.state = { isOpen: true }; 27 | } 28 | 29 | render() { 30 | return ( 31 | 36 | About 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | ); 47 | } 48 | 49 | @bind() 50 | private onDialogClose() { 51 | this.setState({ isOpen: false }); 52 | } 53 | } 54 | 55 | const mapDispatchToProps = (dispatch: Dispatch): DispatchProps => { 56 | return { 57 | onDialogExited: () => { 58 | dispatch(push('/')); 59 | } 60 | }; 61 | }; 62 | 63 | export default connect( 64 | null, 65 | mapDispatchToProps 66 | )(About); 67 | -------------------------------------------------------------------------------- /frontend/src/shared/models/Calendar.ts: -------------------------------------------------------------------------------- 1 | import { GCalApi } from '../google/GCalApi'; 2 | import moment from 'moment'; 3 | 4 | export interface CalendarData { 5 | ical?: string; 6 | web?: string; 7 | scraperId?: string; 8 | googleCalendarId?: string; 9 | } 10 | 11 | export class Calendar { 12 | 13 | private readonly data: CalendarData; 14 | 15 | constructor(data: CalendarData) { 16 | this.data = data; 17 | } 18 | 19 | get events(): Promise { 20 | if (this.data.googleCalendarId) { 21 | return GCalApi.fetchFeed(this.data.googleCalendarId); 22 | } 23 | if (this.data.scraperId) { 24 | const url = `/scraper-data/${this.data.scraperId}.json`; 25 | return fetch(url) 26 | .then((resp) => { 27 | if (!resp.ok) { 28 | throw Error(`${resp.status} ${resp.statusText}`); 29 | } 30 | return resp.json() as Promise; 31 | }) 32 | .then((events) => 33 | events.filter((event) => { 34 | const timeMin = moment().subtract(1, 'day'); 35 | const timeMax = moment().add(1, 'month'); 36 | const eventDate = moment(event.date); 37 | return eventDate.isAfter(timeMin) && eventDate.isBefore(timeMax); 38 | })); 39 | } 40 | return Promise.reject('Feed not available'); 41 | } 42 | 43 | get icalUrl(): string | undefined { 44 | if (this.data.ical) { 45 | return new URL(this.data.ical, window.location.href).toString(); 46 | } else if (this.data.scraperId) { 47 | return new URL(`/scraper-data/${this.data.scraperId}.ics`, window.location.href).toString(); 48 | } else if (this.data.googleCalendarId) { 49 | const cidComponent = encodeURIComponent(this.data.googleCalendarId); 50 | return `https://calendar.google.com/calendar/ical/${cidComponent}/public/basic.ics`; 51 | } 52 | return undefined; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/brooklyn-cb9.py: -------------------------------------------------------------------------------- 1 | import re 2 | from datetime import datetime 3 | 4 | import scrapy 5 | from bs4 import BeautifulSoup 6 | from pytz import timezone 7 | 8 | from cbmap import utils 9 | from cbmap.items import CalEventItem 10 | 11 | 12 | class BrooklynCb9(scrapy.Spider): 13 | name = 'brooklyn-cb9' 14 | title = 'Brooklyn CB9' 15 | 16 | start_urls = [ 17 | 'http://www.communitybrd9bklyn.org/meetings/boardmeetings/', 18 | 'http://www.communitybrd9bklyn.org/meetings/committeemeetings/' 19 | ] 20 | 21 | def parse(self, response): 22 | soup = BeautifulSoup(response.text, 'lxml') 23 | more_info_links = soup.select('.meeting_info a') 24 | for link in more_info_links: 25 | yield scrapy.Request(response.urljoin(link['href']), callback=self.parse_event) 26 | 27 | def parse_event(self, response): 28 | soup = BeautifulSoup(response.text, 'lxml') 29 | 30 | elems = soup.select('.meeting_wrap p') 31 | data = {key.string.rstrip(':').strip(): val.string.strip() 32 | for key, val in (elem.children for elem in elems)} 33 | 34 | date_str = data['Date'] 35 | time_str = data['Time'] 36 | venue = data['Venue'] 37 | address = data['Address'] 38 | 39 | if not date_str or not time_str or time_str == 'CANCELLED': 40 | return 41 | 42 | date_str = utils.strip_date_ords(date_str) 43 | time_str = time_str.replace('.', '') 44 | 45 | event_date = datetime.strptime(date_str, '%B %d, %Y').date() 46 | event_time = self.__parse_time(time_str) 47 | 48 | event_dt = timezone('US/Eastern').localize(datetime.combine(event_date, event_time)) 49 | event_summary = soup.select('.et_main_title')[0].text.strip() 50 | event_description = response.url 51 | event_location = '\n'.join([x for x in (venue, address) if x]) 52 | 53 | yield CalEventItem( 54 | date=event_dt, 55 | summary=event_summary, 56 | description=event_description, 57 | location=event_location 58 | ) 59 | 60 | @staticmethod 61 | def __parse_time(time_str): 62 | try: 63 | return datetime.strptime(time_str, '%I:%M %p').time() 64 | except ValueError: 65 | return datetime.strptime(time_str, '%I:%M%p').time() 66 | -------------------------------------------------------------------------------- /scrapers/cbmap/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | from datetime import date, datetime 3 | from time import mktime 4 | from typing import Optional 5 | 6 | import bleach 7 | from bs4 import BeautifulSoup 8 | from parsedatetime import parsedatetime 9 | from pytz import timezone 10 | 11 | from cbmap.items import CalEventItem 12 | 13 | 14 | ALLOWED_TAGS = [ 15 | 'a', 16 | 'abbr', 17 | 'acronym', 18 | 'b', 19 | 'blockquote', 20 | 'code', 21 | 'em', 22 | 'i', 23 | 'li', 24 | 'ol', 25 | 'strong', 26 | 'ul', 27 | 'p' 28 | ] 29 | 30 | CAL = parsedatetime.Calendar() 31 | 32 | 33 | def parse_calendarjs(response): 34 | def parse_text(text) -> (str, str, str): 35 | event_time = None 36 | soup = BeautifulSoup(text, 'html.parser') 37 | for token in soup.stripped_strings: 38 | result, flag = CAL.parse(token) 39 | if flag == 2: 40 | event_time = datetime.fromtimestamp(mktime(result)).time() 41 | break 42 | return event_time, ' '.join(soup.stripped_strings) 43 | 44 | def parse_date(text) -> date: 45 | return datetime.strptime(text, '%m/%d/%Y').date() # 2/8/2017 46 | 47 | for line in response.text.splitlines(): 48 | if line.startswith('calEvents[calEvents.length]'): 49 | js_str = line.split(' = ')[1].lstrip() 50 | js_str = bytes(js_str, 'utf-8').decode("unicode_escape") 51 | js_str = re.sub(r'[\'"];?$', '', js_str) 52 | js_str = re.sub(r'^[\'"]', '', js_str) 53 | 54 | parts = js_str.split('|') 55 | 56 | event_date = parse_date(parts[0]) 57 | event_time, event_summary = parse_text(parts[1]) 58 | 59 | if event_time: 60 | event_dt = timezone('US/Eastern').localize(datetime.combine(event_date, event_time)) 61 | else: 62 | event_dt = event_date 63 | 64 | yield CalEventItem( 65 | date=event_dt, 66 | summary=event_summary, 67 | location='', 68 | description='' 69 | ) 70 | 71 | 72 | def clean_html(html: str) -> Optional[str]: 73 | if html: 74 | return bleach.clean(html, tags=ALLOWED_TAGS, strip=True) 75 | return None 76 | 77 | 78 | def strip_date_ords(text: str): 79 | return re.sub(r'(?<=\d)(?:th|rd|nd|st)', '', text) -------------------------------------------------------------------------------- /frontend/src/shared/actions/index.ts: -------------------------------------------------------------------------------- 1 | import Location from '../models/Location'; 2 | import { push } from 'react-router-redux'; 3 | import { Dispatch } from 'redux'; 4 | import { ThunkAction } from 'redux-thunk'; 5 | import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; 6 | import { Feature, FeatureCollection, GeoJsonProperties, Polygon } from 'geojson'; 7 | import DISTRICTS_GEOJSON from '../../shared/data/districts-geo.json'; 8 | 9 | export enum ActionType { 10 | SELECT_LOCATION = 'SELECT_LOCATION', 11 | COMPONENT_RESIZED = 'COMPONENT_RESIZED', 12 | } 13 | 14 | interface SelectLocationAction { 15 | type: ActionType.SELECT_LOCATION; 16 | location: Location | null; 17 | } 18 | 19 | interface ComponentResizedAction { 20 | type: ActionType.COMPONENT_RESIZED; 21 | id: string; 22 | size: Size; 23 | } 24 | 25 | export type RootAction = 26 | | SelectLocationAction 27 | | ComponentResizedAction; 28 | 29 | export const selectDistrict = (districtId: number) => { 30 | return push('/districts/' + districtId); 31 | }; 32 | 33 | export const selectLocation = (location: Location | null): ThunkAction => { 34 | return (dispatch: Dispatch) => { 35 | dispatch({ 36 | type: ActionType.SELECT_LOCATION, 37 | location: location 38 | }); 39 | if (location !== null) { 40 | const featureCollection = DISTRICTS_GEOJSON as FeatureCollection; 41 | const districtFeature = featureCollection.features 42 | .find((feature: Feature) => { 43 | return booleanPointInPolygon(location.center, feature.geometry!); 44 | }); 45 | const districtId = (districtFeature && districtFeature.properties) 46 | ? districtFeature.properties.BoroCD : null; 47 | dispatch(selectDistrict(districtId)); 48 | } 49 | }; 50 | }; 51 | 52 | export const clearSelection = () => { 53 | return (dispatch: Dispatch) => { 54 | dispatch(push('/')); 55 | dispatch(selectLocation(null)); 56 | }; 57 | }; 58 | 59 | export const componentResized = (id: string, size: Size): ComponentResizedAction => { 60 | return { 61 | type: ActionType.COMPONENT_RESIZED, 62 | id: id, 63 | size: size 64 | }; 65 | }; 66 | -------------------------------------------------------------------------------- /frontend/src/shared/google/GCalApi.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | 3 | export module GCalApi { 4 | // https://developers.google.com/google-apps/calendar/v3/reference/events/list 5 | export interface EventsList { 6 | items: EventsItem[]; 7 | } 8 | 9 | // https://developers.google.com/google-apps/calendar/v3/reference/events#resource 10 | export interface EventsItem { 11 | id: string; 12 | summary: string; 13 | description: string; 14 | location: string; 15 | start: DateOrDateTime; 16 | end: DateOrDateTime; 17 | } 18 | 19 | interface DateOrDateTime { 20 | date?: string; 21 | dateTime?: string; 22 | } 23 | 24 | export function fetchFeed(calendarId: string): Promise { 25 | return fetch(feedUrl(calendarId)) 26 | .then((resp) => { 27 | if (!resp.ok) { 28 | throw Error(`${resp.status} ${resp.statusText}`); 29 | } 30 | return resp.json(); 31 | }) 32 | .then((json) => json as GCalApi.EventsList) 33 | .then((list) => list.items 34 | .map((item) => ({ 35 | id: item.id, 36 | date: item.start.dateTime || item.start.date!, 37 | summary: item.summary, 38 | location: item.location, 39 | description: item.description 40 | })) 41 | .sort((a, b) => moment(a.date).valueOf() - moment(b.date).valueOf()) 42 | ); 43 | } 44 | 45 | function feedUrl(calendarId: string) { 46 | const calendarIdEncoded = encodeURIComponent(calendarId); 47 | const timeMin = encodeURIComponent(moment().subtract(1, 'day').format()); 48 | const timeMax = encodeURIComponent(moment().add(1, 'month').format()); 49 | return 'https://clients6.google.com/calendar/v3/calendars/' 50 | + calendarId 51 | + '/events' 52 | + `?calendarId=${calendarIdEncoded}` 53 | + '&singleEvents=true' 54 | + '&timeZone=America%2FNew_York' 55 | + '&maxAttendees=1' 56 | + '&maxResults=250' 57 | + '&sanitizeHtml=true' 58 | + `&timeMin=${timeMin}` 59 | + `&timeMax=${timeMax}` 60 | + '&key=AIzaSyBNlYH01_9Hc5S1J9vuFmu2nUqBZJNAXxs'; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/queens-cb3.py: -------------------------------------------------------------------------------- 1 | import scrapy 2 | from bs4 import BeautifulSoup 3 | from icalendar import Calendar 4 | 5 | from cbmap.items import CalEventItem 6 | 7 | 8 | class QueensCb3Spider(scrapy.Spider): 9 | name = 'queens-cb3' 10 | title = 'Queens CB3' 11 | start_urls = [ 12 | 'http://www.cb3qn.nyc.gov/calendar' 13 | ] 14 | 15 | def parse(self, response): 16 | soup = BeautifulSoup(response.text, 'lxml') 17 | 18 | event_links = soup.select('td.feature-cal-inmonth a.feature-cal-inmonth') 19 | for link in event_links: 20 | href = link['href'] 21 | if 'c_eid=' in href: 22 | yield scrapy.Request(response.urljoin(href), callback=self.parse_event) 23 | 24 | depth = response.meta['depth'] or 0 25 | if depth == 0: 26 | next_month_link = soup.find('img', src='/wego/images/next_arrow.gif').parent 27 | request = scrapy.Request(response.urljoin(next_month_link['href'])) 28 | request.meta['depth'] = depth + 1 29 | yield request 30 | 31 | def parse_event(self, response): 32 | soup = BeautifulSoup(response.text, 'lxml') 33 | 34 | ical_link = soup.find('a', text='download calendar file') 35 | 36 | request = scrapy.Request(response.urljoin(ical_link['href']), callback=self.parse_event_ical) 37 | 38 | map_link = soup.find('a', text='Map') 39 | if map_link: 40 | request.meta['location'] = ', '.join( 41 | (x.string.strip() 42 | for x in reversed(list(map_link.previous_siblings)) 43 | if x.string and x.string.strip())) 44 | 45 | yield request 46 | 47 | def parse_event_ical(self, response): 48 | cal = Calendar.from_ical(response.body_as_unicode()) 49 | for vevent in cal.subcomponents: 50 | event_dt = vevent.get('DTSTART').dt 51 | event_summary = vevent.get('SUMMARY') 52 | event_description = vevent.get('DESCRIPTION') 53 | event_location = response.meta.get('location', None) 54 | 55 | if event_summary == 'Alternate Side Parking Rules Suspended': 56 | continue 57 | 58 | yield CalEventItem( 59 | date=event_dt, 60 | summary=event_summary, 61 | description=event_description, 62 | location=event_location 63 | ) 64 | 65 | -------------------------------------------------------------------------------- /scrapers/cbmap/spiders/brooklyn-cb6.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import scrapy 4 | from bs4 import BeautifulSoup 5 | from pytz import timezone 6 | 7 | from cbmap import utils 8 | from cbmap.items import CalEventItem 9 | 10 | 11 | class BrooklynCb6Spider(scrapy.Spider): 12 | name = 'brooklyn-cb6' 13 | title = 'Brooklyn CB6' 14 | start_urls = [ 15 | 'http://www1.nyc.gov/site/brooklyncb6/calendar/calendar.page' 16 | ] 17 | 18 | def parse(self, response): 19 | soup = BeautifulSoup(response.text, 'lxml') 20 | 21 | for tag in soup.select('.about-description > h3'): 22 | # Find index of next event header. 23 | all_siblings = tag.select('~ *') 24 | next_header = tag.select_one('~ hr, ~ h3, ~ h2') 25 | 26 | # Find all siblings up to next header (or end of document if last event). 27 | event_tags = tag.select('~ *', limit=all_siblings.index(next_header)) \ 28 | if next_header else all_siblings 29 | 30 | # Use

text as date. 31 | event_date = self.__parse_date(tag.string) 32 | 33 | # Find first

sibling. 34 | event_summary = next((t.string for t in event_tags if t.name == 'h4'), None) 35 | 36 | # If not found, look for first

sibling where all children are . 37 | if not event_summary: 38 | event_summary = next((t.string for t in event_tags 39 | if t.name == 'p' and t.string and all(c.name == 'b' for c in t.children)), None) 40 | 41 | # Find first

sibling where all children are not . 42 | event_location = next((', '.join(t.stripped_strings) for t in event_tags 43 | if t.name == 'p' and all(c.name != 'b' for c in t.children)), None) 44 | 45 | # Find first