├── .gitignore ├── agency ids.csv ├── README.md ├── add_collaborator.py ├── export_response_tasks.py ├── LICENSE ├── export_jurisdiction_stats.py ├── export_tasks.py ├── utils.py ├── export_documentcloud_text.py ├── embargo_all_user_requests.py ├── export_file_data.py ├── transfer_all_user_requests.py ├── file.py ├── crest-diaries.py ├── unembargo_all_user_requests.py ├── unembargo_all_original.py ├── admin ├── import_draft_completed_agency_tasks.py └── export_open_agency_tasks.py ├── upload_responsive_documents_to_documentcloud.py ├── export_assignment_responses.py ├── multirequest.py ├── export_news.py ├── export.py ├── export_stats.py ├── dogfood ├── sundayreminder.py ├── mondayreport.py └── monday_reloaded.py ├── export_as_files.py ├── bulk_file.py ├── export_agency_stats.py ├── export_all_requests_as_files.py ├── export_standard_full.py ├── export_all_user_requests.py ├── export_users_requests_as_files.py ├── summarize_open_requests.py └── export_foia.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.DS_Store 3 | .DS_Store 4 | credentials.py 5 | -------------------------------------------------------------------------------- /agency ids.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MuckRock/API-examples/HEAD/agency ids.csv -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | MuckRock API 2 | ============ 3 | 4 | This repository presents a collection of scripts demonstrating the MuckRock API. Find more information on [filing requests in bulk](https://muckrock.notion.site/Guides-and-Manuals-9852bdcb6046454f80840a3f833fd95c) on our Notion guide. 5 | 6 | The API has a [visual interface](https://www.muckrock.com/api_v1/), but it is most powerful when accessed programmatically. 7 | 8 | To access the API, you must first get your API token, which is [currently available on your profile page](https://www.muckrock.com/accounts/profile/) (scroll to the bottom left). The key is generated from your MuckRock user account and password. If you don't have an account, [sign up here](https://www.muckrock.com/accounts/register). 9 | -------------------------------------------------------------------------------- /add_collaborator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | 5 | import requests 6 | 7 | import utils 8 | 9 | url = utils.API_URL 10 | token = utils.get_api_key() 11 | headers = utils.get_headers(token) 12 | 13 | # set the IDs for the requests to change here 14 | FOIA_IDS = [] 15 | # set the IDs for the users to add as editors here 16 | EDITORS = [] 17 | # set the IDs for the users to add as viewers here 18 | VIEWERS = [] 19 | 20 | for foia_id in FOIA_IDS: 21 | print(f"Updating request {foia_id}") 22 | response = requests.patch( 23 | f"{url}foia/{foia_id}/", 24 | headers=headers, 25 | json={ 26 | "edit_collaborators": EDITORS, 27 | "read_collaborators": VIEWERS, 28 | }, 29 | ) 30 | response.raise_for_status() 31 | -------------------------------------------------------------------------------- /export_response_tasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | import utils 7 | 8 | url = utils.API_URL 9 | token = utils.get_api_key() 10 | headers = utils.get_headers(token) 11 | 12 | next_ = url + 'responsetask' 13 | 14 | fields = ( 15 | 'id', 16 | 'resolved_by', 17 | 'communication', 18 | 'date_created', 19 | 'date_done', 20 | 'resolved', 21 | 'created_from_orphan', 22 | 'predicted_status', 23 | 'predicted_status', 24 | ) 25 | 26 | page = 1 27 | 28 | csv_file = open('tasks.csv', 'w') 29 | csv_file.seek(0) 30 | csv_writer = unicodecsv.writer(csv_file) 31 | csv_writer.writerow(fields) 32 | 33 | while next_ is not None: 34 | r = requests.get(next_, headers=headers) 35 | try: 36 | json = r.json() 37 | next_ = json['next'] 38 | for datum in json['results']: 39 | csv_writer.writerow([datum[field] for field in fields]) 40 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 41 | page += 1 42 | except: 43 | print r 44 | print r.text 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 MuckRock 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /export_jurisdiction_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | from utils import get_api_key 7 | 8 | token = get_api_key() 9 | url = 'https://www.muckrock.com/api_v1/' 10 | 11 | headers = {'Authorization': 'Token %s' % token, 'content-type': 'application/json'} 12 | next_ = url + 'jurisdiction' 13 | 14 | fields = ( 15 | "id", 16 | "name", 17 | "slug", 18 | "full_name", 19 | "level", 20 | "parent", 21 | "days", 22 | "absolute_url", 23 | "average_response_time", 24 | "fee_rate", 25 | "success_rate" 26 | ) 27 | 28 | page = 1 29 | 30 | csv_file = open('jurisdiction_stats.csv', 'w') 31 | csv_file.seek(0) 32 | csv_writer = unicodecsv.writer(csv_file) 33 | csv_writer.writerow(fields) 34 | 35 | while next_ is not None: 36 | r = requests.get(next_, headers=headers) 37 | try: 38 | json = r.json() 39 | next_ = json['next'] 40 | for datum in json['results']: 41 | csv_writer.writerow([datum[field] for field in fields]) 42 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 43 | page += 1 44 | except Exception as e: 45 | print e 46 | -------------------------------------------------------------------------------- /export_tasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | import utils 7 | 8 | url = utils.API_URL 9 | token = utils.get_api_key() 10 | headers = utils.get_headers(token) 11 | 12 | next_ = url + 'task' 13 | 14 | fields = ( 15 | 'id', 16 | 'date_created', 17 | 'date_done', 18 | 'resolved', 19 | 'assigned', 20 | 'orphantask', 21 | 'snailmailtask', 22 | 'rejectedemailtask', 23 | 'staleagencytask', 24 | 'flaggedtask', 25 | 'newagencytask', 26 | 'responsetask', 27 | ) 28 | 29 | page = 1 30 | 31 | csv_file = open('tasks.csv', 'w') 32 | csv_file.seek(0) 33 | csv_writer = unicodecsv.writer(csv_file) 34 | csv_writer.writerow(fields) 35 | 36 | while next_ is not None: 37 | r = requests.get(next_, headers=headers) 38 | try: 39 | json = r.json() 40 | next_ = json['next'] 41 | for datum in json['results']: 42 | csv_writer.writerow([datum[field] for field in fields]) 43 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 44 | page += 1 45 | except: 46 | print r 47 | print r.text 48 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import os 5 | import sys 6 | from getpass import getpass 7 | 8 | import requests 9 | 10 | # Using this method of secret storing: 11 | # https://stackoverflow.com/questions/25501403/storing-the-secrets-passwords-in-a-separate-file 12 | 13 | try: 14 | import credentials 15 | token = credentials.API_key 16 | except Exception as e: 17 | print (e) 18 | token = getpass("API Token: ") 19 | credentials_file = open("credentials.py", "w") 20 | credentials_file.writelines("API_key = \"" + token +"\"") 21 | credentials_file.close 22 | 23 | 24 | API_URL = 'https://www.muckrock.com/api_v1/' 25 | 26 | def get_api_key(): 27 | return token 28 | 29 | def get_headers(token=None): 30 | if token: 31 | return { 32 | 'Authorization': 'Token %s' % token, 33 | 'content-type': 'application/json' 34 | } 35 | else: 36 | return {'content-type': 'application/json'} 37 | 38 | def display_progress(current, total): 39 | percent = (current / total) * 100.00 40 | sys.stdout.write("\r%d%%" % percent) 41 | sys.stdout.flush() 42 | -------------------------------------------------------------------------------- /export_documentcloud_text.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | from utils import get_api_key 7 | 8 | from documentcloud import DocumentCloud 9 | client = DocumentCloud() 10 | 11 | client = DocumentCloud(USERNAME, PASSWORD) 12 | 13 | # request_pks = ["296716-request-denied"] 14 | 15 | csv_file = open('document_cloud_OCR.csv', 'w') 16 | csv_file.seek(0) 17 | csv_writer = unicodecsv.writer(csv_file) 18 | csv_writer.writerow(["ID","OCR Text"]) 19 | 20 | with open('docIDs.csv', mode='r') as infile: 21 | reader = unicodecsv.reader(infile) 22 | with open('docIDs.csv', mode='r') as outfile: 23 | print "starting to write" 24 | writer = unicodecsv.writer(outfile) 25 | print "starting to read" 26 | request_pks = {rows[0]:rows[0] for rows in reader} 27 | print "all read" 28 | print request_pks 29 | 30 | 31 | for document_id in request_pks: 32 | print "Working on request " + str(document_id) 33 | try: 34 | obj = client.documents.get(document_id) 35 | OCR = obj.full_text 36 | csv_writer.writerow([document_id,OCR]) 37 | except Exception as e: 38 | print e 39 | -------------------------------------------------------------------------------- /embargo_all_user_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests as http_request 5 | import json 6 | import utils 7 | 8 | url = utils.API_URL 9 | token = utils.get_api_key() 10 | headers = utils.get_headers(token) 11 | 12 | username = raw_input('Username: ') 13 | next_url = "https://www.muckrock.com/api_v1/foia/?user=" + username 14 | current_page = 0 15 | 16 | while next_url: 17 | # we use next_url because the API results are paginated 18 | r = http_request.get(next_url, headers=headers) 19 | data = r.json() 20 | next_url = data['next'] 21 | 22 | # measures progress by page, not by result 23 | current_page += 1 24 | total_pages = (data['count'] / 20.0) 25 | utils.display_progress(current_page, total_pages) 26 | 27 | for request in data['results']: 28 | request_id = request['id'] 29 | request_url = 'https://www.muckrock.com/api_v1/foia/%s/' % str(request_id) 30 | # Removes the embargo date to make sure it permanently embargos. 31 | data = json.dumps({ 32 | 'embargo': True, 33 | 'date_embargo': None, 34 | 'permanent_embargo': True 35 | }) 36 | http_request.patch(request_url, headers=headers, data=data) 37 | -------------------------------------------------------------------------------- /export_file_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | from utils import get_api_key 7 | 8 | token = get_api_key() 9 | url = 'https://www.muckrock.com/api_v1/' 10 | 11 | headers = {'Authorization': 'Token %s' % token, 'content-type': 'application/json'} 12 | next_ = url + 'communication' 13 | 14 | fields = ( 15 | "id", 16 | "ffile", 17 | "title", 18 | "date", 19 | "source", 20 | "description", 21 | "access", 22 | "doc_id", 23 | "pages" 24 | ) 25 | 26 | page = 1 27 | 28 | csv_file = open('doccloudfiles.csv', 'w') 29 | csv_file.seek(0) 30 | csv_writer = unicodecsv.writer(csv_file) 31 | csv_writer.writerow(fields) 32 | 33 | while next_ is not None: 34 | r = requests.get(next_, headers=headers) 35 | try: 36 | json = r.json() 37 | next_ = json['next'] 38 | for datum in json['results']: 39 | print "datum" 40 | for docfile in datum['files']: 41 | print docfile 42 | csv_writer.writerow([docfile[field] for field in fields]) 43 | print "docfile" 44 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 45 | page += 1 46 | except Exception as e: 47 | print e 48 | -------------------------------------------------------------------------------- /transfer_all_user_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests as http_request 5 | import json 6 | import utils 7 | 8 | url = utils.API_URL 9 | token = utils.get_api_key() 10 | headers = utils.get_headers(token) 11 | 12 | 13 | 14 | username = raw_input('Which user do you want to remove requests from?: ') 15 | destination = raw_input('Destination user ID (not username): ') 16 | 17 | next_url = url + "foia/?user=" + username 18 | current_page = 0 19 | 20 | while next_url: 21 | # we use next_url because the API results are paginated 22 | r = http_request.get(next_url, headers=headers) 23 | data = r.json() 24 | next_url = data['next'] 25 | 26 | # measures progress by page, not by result 27 | current_page += 1 28 | total_pages = (data['count'] / 20.0) 29 | utils.display_progress(current_page, total_pages) 30 | 31 | for request in data['results']: 32 | print "Working on request " + str(request['id']) 33 | request_id = request['id'] 34 | request_url = 'https://www.muckrock.com/api_v1/foia/%s/' % str(request_id) 35 | print "This is the request url " + request_url 36 | data = json.dumps({ 37 | 'user': destination 38 | }) 39 | print data 40 | editedRequest = http_request.patch(request_url, headers=headers, data=data) 41 | -------------------------------------------------------------------------------- /file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import json 6 | import utils 7 | 8 | import csv 9 | 10 | with open('agencies.csv', 'rb') as f: 11 | reader = csv.reader(f) 12 | agencies = list(reader) 13 | 14 | url = utils.API_URL 15 | token = utils.get_api_key() 16 | headers = utils.get_headers(token) 17 | 18 | f = open("requestLanguage.txt","r") 19 | message = f.read() 20 | 21 | print message # Just to help verify that the language is what you want included. 22 | 23 | # If you'd like to file a test request, test agency is 248. Note that this uses a request, which you can ask and we'll refund. 24 | 25 | for agency in agencies: 26 | print "Filing with " + str(agency) 27 | data = json.dumps({ 28 | 'agency': agency, 29 | 'title': 'My Public Records Request', 30 | 'full_text': message, # Use if you don't want any MuckRock preamble/post-amble text added. 31 | # 'document_request': message, # use if you do want template added around your message text 32 | # 'attachments': ['https://muckrock.s3.us-east-1.amazonaws.com/files_static/some_attachment.pdf'], 33 | 'permanent_embargo': True 34 | # 'embargo': True 35 | }) 36 | # The request will be saved as a draft if you do not have any requests left 37 | r = requests.post(url + 'foia/', headers=headers, data=data) 38 | print(r) 39 | -------------------------------------------------------------------------------- /crest-diaries.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import csv 3 | from datetime import datetime 4 | import os.path 5 | 6 | # To do: 7 | # - Limit to X updates per day 8 | # - Post tweets! 9 | # - Screenshot 10 | # - ??? 11 | 12 | csvNumber = 1 13 | # deduplicate = True 14 | 15 | print "Kicking off" 16 | 17 | while csvNumber < 5: 18 | if not os.path.isfile('crest_lite_' + str(csvNumber) + '.csv'): 19 | urllib.urlretrieve('https://cdn.muckrock.com/files_static/crest_csv/crest_lite_' + str(csvNumber) + '.csv', 'crest_lite_' + str(csvNumber) + '.csv') 20 | print "We downloaded the CSV." 21 | else: 22 | print "We already have that CSV." 23 | with open('crest_lite_' + str(csvNumber) + '.csv') as csvfile: 24 | print "Opening Crest Lite " + (str(csvNumber)) 25 | reader = csv.DictReader(csvfile) 26 | for row in reader: 27 | if row['publication_date'] != '' and 'Diary' in row['title']: 28 | print "Found a diary!" 29 | datetime_object = datetime.strptime(row['publication_date'], '%B %d, %Y') 30 | csv_file = open('diary ' + str(datetime_object.strftime('%m')) + '-' + str(datetime_object.strftime('%d')) + '-' + str(datetime_object.strftime('%Y')) + '.csv', 'a') 31 | csv_writer = csv.writer(csv_file) 32 | csv_writer.writerow( 33 | # else: 34 | # print "No date or not a diary" 35 | print "All done." 36 | csvNumber += 1 37 | -------------------------------------------------------------------------------- /unembargo_all_user_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | import requests 4 | import json 5 | import utils 6 | 7 | url = utils.API_URL 8 | token = utils.get_api_key() 9 | headers = utils.get_headers(token) 10 | 11 | user = "Example" #<--- Put username here. Case sensitive. 12 | 13 | page = 1 14 | next_ = url+"foia/?user="+user 15 | 16 | 17 | 18 | while next_ is not None: 19 | r = requests.get(next_, headers=headers) 20 | try: 21 | json_data = r.json() 22 | print 'Page %d of %d' % (page, json_data['count'] / 20 + 1) 23 | next_ = json_data['next'] 24 | for request in json_data['results']: 25 | reqNumber = request["id"] 26 | editedRequest = requests.get(url + 'foia/%s/' % str(reqNumber), headers=headers) 27 | print "Embargoing request number " + str(reqNumber) + " for " + request["title"] + " filed by " + request["username"] 28 | 29 | data = json.dumps({ 30 | 'embargo': False, 31 | 'date_embargo': None, #Removes the embargo date to make sure it actually embargos. 32 | }) 33 | editedRequest = requests.patch(url + 'foia/%d/' % reqNumber, headers=headers, data=data) 34 | 35 | print "Request Unembargoed." 36 | if editedRequest.status_code != 200: 37 | print '*In Ron Burgendy Voice:* Error? ', editedRequest.status_code, r.text 38 | page += 1 39 | except Exception as e: 40 | print e 41 | print 'Error! ', editedRequest.status_code, ": ", editedRequest.text 42 | -------------------------------------------------------------------------------- /unembargo_all_original.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | import requests 4 | import json 5 | import utils 6 | 7 | url = utils.API_URL 8 | token = utils.get_api_key() 9 | headers = utils.get_headers(token) 10 | 11 | user = "NYWFOIL" #<--- Put username here. Case sensitive. 12 | 13 | page = 1 14 | next_ = url+"foia/?user="+user 15 | 16 | 17 | 18 | while next_ is not None: 19 | print "URL I'm using is " + next_ 20 | r = requests.get(next_, headers=headers) 21 | print "hey rob. check out " + str(r) 22 | try: 23 | print "Getting that sweet JSON" 24 | json_data = r.json() 25 | print 'Page %d of %d' % (page, json_data['count'] / 20 + 1) 26 | next_ = json_data['next'] 27 | for request in json_data['results']: 28 | reqNumber = request["id"] 29 | editedRequest = requests.get(url + 'foia/%s/' % str(reqNumber), headers=headers) 30 | print "Embargoing request number " + str(reqNumber) + " for " + request["title"] + " filed by " + request["username"] 31 | 32 | data = json.dumps({ 33 | 'embargo': False, 34 | 'date_embargo': None, #Removes the embargo date to make sure it actually embargos. 35 | }) 36 | editedRequest = requests.patch(url + 'foia/%d/' % reqNumber, headers=headers, data=data) 37 | 38 | print "Request Embargoed." 39 | if editedRequest.status_code != 200: 40 | print '*In Ron Burgendy Voice:* Error? ', editedRequest.status_code, r.text 41 | page += 1 42 | except Exception as e: 43 | print e 44 | print 'Error! ', editedRequest.status_code, ": ", editedRequest.text -------------------------------------------------------------------------------- /admin/import_draft_completed_agency_tasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | import utils 7 | import json 8 | 9 | 10 | url = utils.API_URL 11 | token = utils.get_api_key() 12 | headers = utils.get_headers(token) 13 | 14 | page = 1 15 | 16 | with open('agency_upload.csv') as csvfile: 17 | reader = unicodecsv.DictReader(csvfile) 18 | for row in reader: 19 | try: 20 | agency_url = url+"agency/"+str(row["Input.agency"])+"/" 21 | print agency_url 22 | a = requests.get(agency_url, headers=headers) 23 | agency_json = a.json() 24 | print "We're looking up new agency info for " + agency_json["name"] 25 | if agency_json['status'] == "pending" and agency_json['phone'] == '' and agency_json['email'] == '' and agency_json['address'] == '' and agency_json['fax'] == '' and agency_json['website'] == '':# Make sure no one has done the agency in the meantime 26 | print "Doing the R thang" 27 | r = requests.get(agency_url, headers=headers) 28 | data = json.dumps({ 29 | 'address': row["Answer.address"], 30 | 'email': row["Answer.email"], 31 | 'fax': row["Answer.fax"], 32 | 'phone': row["Answer.phone"], 33 | 'website': row["Answer.website"] 34 | }) 35 | print agency_url 36 | requests.patch(agency_url, headers=headers, data=data) 37 | else: 38 | print "That agency has already been tweaked" 39 | except: 40 | print "Error with agency " + row["Input.agency"] + ", usually because it was deleted." 41 | -------------------------------------------------------------------------------- /upload_responsive_documents_to_documentcloud.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sample script that uses the MuckRock and DocumentCloud APIs 3 | to upload all of the documents from a MuckRock FOIA request 4 | to a project on DocumentCloud 5 | 6 | """ 7 | import os 8 | from muckrock import MuckRock 9 | from documentcloud import DocumentCloud 10 | 11 | # Safely store our credentials in local environment variables 12 | # so that we don't expose them to the internet 13 | ma_user = os.environ.get("MA_USER") 14 | ma_pass = os.environ.get("MA_PASSWORD") 15 | 16 | # Start our MuckRock requests client 17 | mr_client = MuckRock(ma_user, ma_pass) 18 | 19 | # Prompt the user to enter a request ID they'd like to capture documents from 20 | request_id = input("Please enter the request ID you'd like to download: ") 21 | request = mr_client.requests.retrieve(request_id) 22 | 23 | # Retrieves all of the communications from the request 24 | comms_list = request.get_communications() 25 | 26 | all_files = [] 27 | 28 | # For each communication, if there is an attached file, add the file to a list. 29 | for comm in comms_list: 30 | files = list(comm.get_files()) 31 | if files: # Filters out comms with no actual files 32 | all_files.extend(files) 33 | 34 | # Build out a list of file URLs we can use to upload to DocumentCloud 35 | file_urls = [] 36 | for file in all_files: 37 | file_urls.append(file.ffile) 38 | 39 | # Start our DocumentCloud client 40 | dc_client = DocumentCloud(ma_user, ma_pass) 41 | 42 | # Prompt our user for a project name 43 | project_name = input("What would you like to name this project?") 44 | 45 | # Create the project 46 | project = dc_client.projects.create(project_name) 47 | 48 | # Upload all of the files to our DocumentCloud project 49 | our_docs = dc_client.documents.upload_urls(file_urls, projects=[project.id]) 50 | -------------------------------------------------------------------------------- /export_assignment_responses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | from utils import get_api_key 7 | 8 | token = get_api_key() 9 | url = 'https://www.muckrock.com/api_v1/' 10 | 11 | headers = {'Authorization': 'Token %s' % token, 'content-type': 'application/json'} 12 | 13 | 14 | assignmentID = raw_input('What assignment are you trying to export? Look at the numbers at the end of the URL: ') 15 | next_ = url + "assignment-responses/?crowdsource=" + str(assignmentID) 16 | 17 | basic_fields = [ 18 | "id", 19 | "user", 20 | "datetime", 21 | "tags", 22 | "skip", 23 | "number", 24 | "flag", 25 | "gallery", 26 | "crowdsource" 27 | ] 28 | 29 | assignment_fields = [] 30 | 31 | page = 1 32 | 33 | csv_file = open('assignment ' + str(assignmentID) + ' export.csv', 'w') 34 | csv_file.seek(0) 35 | csv_writer = unicodecsv.writer(csv_file) 36 | 37 | 38 | r = requests.get(next_, headers=headers) 39 | json = r.json() 40 | 41 | for column in json['results'][0]['values']: 42 | assignment_fields.append(column['field']) 43 | csv_writer.writerow(basic_fields + assignment_fields) 44 | 45 | while next_ is not None: 46 | r = requests.get(next_, headers=headers) 47 | try: 48 | json = r.json() 49 | next_ = json['next'] 50 | for datum in json['results']: 51 | submissions = [] 52 | for field in basic_fields: 53 | submissions.append(datum[field]) 54 | for entry in datum["values"]: 55 | submissions.append(entry["value"]) 56 | csv_writer.writerow(submissions) 57 | # csv_writer.writerow(datum[field] for field in basic_fields + ) 58 | # + datum['values'][assignment_fields] for field in assignment_fields 59 | print 'Page %d of %d' % (page, json['count'] / 40) 60 | page += 1 61 | except Exception as e: 62 | print e 63 | -------------------------------------------------------------------------------- /multirequest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import json 6 | import utils 7 | 8 | url = utils.API_URL 9 | token = utils.get_api_key() 10 | headers = utils.get_headers(token) 11 | 12 | EXISTING_PKS = [] # If you've already filed with a given agency, or otherwise want to exclude it, include its ID here. 13 | 14 | 15 | AGENCY_PKS = [248] # Agency ID 248 is a test agency under federal jurisdiction 10. This ID is subject to change, and will deduct requests from your account. Contact info@muckrock.com and we'll add them back. 16 | 17 | AGENCY_PKS = filter(lambda x: x not in EXISTING_PKS, AGENCY_PKS) 18 | DOCS = """ 19 | A copy of your reports that are: 20 | Annual 21 | Monthly 22 | Bimonthly 23 | """ 24 | TITLE = 'Records Request' # Customize here for your project 25 | 26 | for agency_pk in AGENCY_PKS: 27 | # get the jurisdiction 28 | r = requests.get(url + 'agency/{}/'.format(agency_pk), headers=headers) 29 | jurisdiction_pk = r.json()['jurisdiction'] 30 | 31 | print 'Filing for {}...'.format(r.json()['name']) 32 | 33 | # get the template text 34 | r = requests.get(url + 'jurisdiction/{}/template/'.format(jurisdiction_pk), headers=headers) 35 | text = r.json()['text'] 36 | 37 | # replace the text as required, such as if requests are permanently embargoed 38 | text = text.replace('', DOCS) 39 | text = text.replace( 40 | 'The requested documents will be made available to the general public, and t', 41 | 'T', 42 | ) 43 | print "Request Text: " + text 44 | # file the request 45 | data = json.dumps({ 46 | 'jurisdiction': jurisdiction_pk, 47 | 'agency': agency_pk, 48 | 'title': TITLE, 49 | 'full_text': text, 50 | 'requested_docs': DOCS, 51 | 'embargo': True, 52 | }) 53 | r = requests.post(url + 'foia/', headers=headers, data=data) 54 | -------------------------------------------------------------------------------- /export_news.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | ## Need to install google-api-python-clientself. 5 | ## pip install --upgrade google-api-python-client 6 | 7 | import requests 8 | import unicodecsv 9 | from utils import get_api_key 10 | from dateutil import parser 11 | 12 | # GA_product = raw_input('What is your Google Analytics product Key? ') 13 | 14 | 15 | token = get_api_key() 16 | url = 'https://www.muckrock.com/api_v1/' 17 | 18 | headers = {'Authorization': 'Token %s' % token, 'content-type': 'application/json'} 19 | next_ = url + 'news' 20 | 21 | fields = ( 22 | "id", 23 | "authors", 24 | "editors", 25 | "pub_date", 26 | "title", 27 | "kicker", 28 | "slug", 29 | "summary", 30 | "body", 31 | "publish", 32 | "image" 33 | ) 34 | 35 | field_names = ( 36 | "id", 37 | "authors", 38 | "editors", 39 | "pub_date", 40 | "title", 41 | "kicker", 42 | "slug", 43 | "summary", 44 | "body", 45 | "publish", 46 | "image", 47 | "wordcount", 48 | "title_wordcount", 49 | "full_url" 50 | ) 51 | 52 | page = 1 53 | 54 | csv_file = open('news.csv', 'w') 55 | csv_file.seek(0) 56 | csv_writer = unicodecsv.writer(csv_file) 57 | csv_writer.writerow(field_names) 58 | 59 | while next_ is not None: 60 | r = requests.get(next_, headers=headers) 61 | try: 62 | json = r.json() 63 | next_ = json['next'] 64 | for datum in json['results']: 65 | pack = [datum[field] for field in fields] 66 | datum['wordcount'] = len(datum['body'].split()) 67 | datum['title_wordcount'] = len(datum['title'].split()) 68 | datum['full_url'] = 'https://www.muckrock.com/news/archives/' + parser.parse(datum['pub_date']).strftime('%Y') + '/' + parser.parse(datum['pub_date']).strftime('%b').lower() + '/' + parser.parse(datum['pub_date']).strftime('%d') + '/' + datum['slug'] + '/' 69 | print datum['full_url'] 70 | csv_writer.writerow([datum[field] for field in field_names]) 71 | print 'Page %d of %d' % (page, (json['count'] / 50) + 1) 72 | page += 1 73 | except Exception as e: 74 | print e 75 | -------------------------------------------------------------------------------- /export.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import urllib, os, json, datetime, requests, urlparse 5 | import utils 6 | 7 | url = utils.API_URL 8 | token = utils.get_api_key() 9 | headers = utils.get_headers(token) 10 | 11 | # Stuff borrowed from http://stackoverflow.com/questions/6373094/how-to-download-a-file-to-a-specific-path-in-the-server-python 12 | 13 | request_pks = [10565] 14 | 15 | for pk in request_pks: 16 | print "Working on request " + str(pk) 17 | r = requests.get(url + 'foia/%d/' % pk, headers=headers) 18 | json_data = r.json() 19 | communications = json_data['communications'] 20 | 21 | if communications is None: 22 | print "It looks like there were no communications here." 23 | 24 | if not os.path.exists(str(pk)): # Checks to see if the folder exists. 25 | dirName = str(pk) 26 | print "Creating directory /" + str(pk) 27 | os.makedirs(str(pk)) 28 | else: 29 | print "The directory already exists. Phew." 30 | 31 | 32 | 33 | for communication in communications: 34 | commNum = 0 35 | #print communication 36 | for file in communication['files']: 37 | print "Trying to grab a file from communication " + str(commNum) 38 | url = file['ffile'] 39 | split = urlparse.urlsplit(url) # grabbed from http://stackoverflow.com/questions/2795331/python-download-without-supplying-a-filename 40 | filename = split.path.split("/")[-1] 41 | filename = str(communication["date"])+" "+filename 42 | print filename 43 | #urllib.urlretrieve(url, '/'+str(pk)+'/'+filename) 44 | urllib.urlretrieve(url, str(pk) + '/' + filename) 45 | 46 | 47 | print "Trying to grab the text from the communication" 48 | # eventually this should save to pdf, maybe using this: https://github.com/mstamy2/PyPDF2/tree/master/Sample_Code 49 | 50 | communicationText = communication["communication"].encode('ascii', 'ignore') 51 | text_file = open(str(pk) + '/' + communication["date"] + " Communication.txt", "w+") 52 | text_file.write(communicationText) 53 | text_file.close() 54 | -------------------------------------------------------------------------------- /export_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | from utils import get_api_key 7 | 8 | token = get_api_key() 9 | url = 'https://www.muckrock.com/api_v1/' 10 | 11 | headers = {'Authorization': 'Token %s' % token, 'content-type': 'application/json'} 12 | next_ = url + 'statistics' 13 | 14 | fields = ( 15 | "date", 16 | "total_requests", 17 | "total_requests_success", 18 | "total_requests_denied", 19 | "total_requests_draft", 20 | "total_requests_submitted", 21 | "total_requests_awaiting_ack", 22 | "total_requests_awaiting_response", 23 | "total_requests_awaiting_appeal", 24 | "total_requests_fix_required", 25 | "total_requests_payment_required", 26 | "total_requests_no_docs", 27 | "total_requests_partial", 28 | "total_requests_abandoned", 29 | "total_pages", 30 | "total_users", 31 | "total_agencies", 32 | "total_fees", 33 | "pro_users", 34 | "pro_user_names", 35 | "total_page_views", 36 | "daily_requests_pro", 37 | "daily_requests_basic", 38 | "daily_requests_beta", 39 | "daily_articles", 40 | "total_tasks", 41 | "total_unresolved_tasks", 42 | "total_generic_tasks", 43 | "total_unresolved_generic_tasks", 44 | "total_orphan_tasks", 45 | "total_unresolved_orphan_tasks", 46 | "total_snailmail_tasks", 47 | "total_unresolved_snailmail_tasks", 48 | "total_rejected_tasks", 49 | "total_unresolved_rejected_tasks", 50 | "total_staleagency_tasks", 51 | "total_unresolved_staleagency_tasks", 52 | "total_flagged_tasks", 53 | "total_unresolved_flagged_tasks", 54 | "total_newagency_tasks", 55 | "total_unresolved_newagency_tasks", 56 | "total_response_tasks", 57 | "total_unresolved_response_tasks", 58 | "total_faxfail_tasks", 59 | "total_unresolved_faxfail_tasks", 60 | "total_payment_tasks", 61 | "total_unresolved_payment_tasks", 62 | "total_crowdfundpayment_tasks", 63 | "total_unresolved_crowdfundpayment_tasks" 64 | ) 65 | 66 | page = 1 67 | 68 | csv_file = open('stats.csv', 'w') 69 | csv_file.seek(0) 70 | csv_writer = unicodecsv.writer(csv_file) 71 | csv_writer.writerow(fields) 72 | 73 | while next_ is not None: 74 | r = requests.get(next_, headers=headers) 75 | try: 76 | json = r.json() 77 | next_ = json['next'] 78 | for datum in json['results']: 79 | csv_writer.writerow([datum[field] for field in fields]) 80 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 81 | page += 1 82 | except Exception as e: 83 | print e 84 | -------------------------------------------------------------------------------- /admin/export_open_agency_tasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | import utils 7 | 8 | url = utils.API_URL 9 | token = utils.get_api_key() 10 | headers = utils.get_headers(token) 11 | 12 | next_ = url + 'newagencytask' 13 | 14 | task_fields = ( 15 | 'id', 16 | 'agency', 17 | 'resolved' 18 | ) 19 | 20 | agency_fields = ( 21 | 'id', 22 | 'name', 23 | 'status', 24 | 'jurisdiction', 25 | 'address', 26 | 'email', 27 | 'other_emails', 28 | 'phone', 29 | 'fax', 30 | 'website' 31 | ) 32 | 33 | page = 1 34 | 35 | csv_file = open('incomplete_agencies.csv', 'w') 36 | csv_file.seek(0) 37 | csv_writer = unicodecsv.writer(csv_file) 38 | csv_writer.writerow(task_fields + agency_fields + ("Jurisdiction Name",)) 39 | 40 | print "OK, set up the file" 41 | 42 | while next_ is not None: 43 | r = requests.get(next_, headers=headers) 44 | try: 45 | json = r.json() 46 | next_ = json['next'] 47 | for datum in json['results']: 48 | print "The task " + str(datum["id"]) + " is " + str(datum["resolved"]) 49 | if not (datum["resolved"]): 50 | a = requests.get((url+"agency/"+str(datum["agency"])), headers=headers) 51 | agency_json = a.json() 52 | print "We're looking up new agency info for " + agency_json["name"] 53 | all_fields = dict(datum) 54 | all_fields.update(agency_json) 55 | print "Created all fields. whatever the hell that is" 56 | print "Looking up jurisdiction " + str(agency_json["jurisdiction"]) 57 | print url+"jurisdiction/"+str(agency_json["jurisdiction"]) 58 | j = requests.get((url+"jurisdiction/"+str(agency_json["jurisdiction"])), headers=headers) 59 | jurisdiction_json = j.json() 60 | if jurisdiction_json["full_name"] is True: 61 | jurisdiction2 = jurisdiction_json["full_name"] 62 | print "The jurisdiction's full name is " + jurisdiction2 63 | else: 64 | jurisdiction2 = jurisdiction_json["name"] 65 | print "The jurisdiction is a state and its full name is " + jurisdiction2 66 | all_fields.update({'jurisdiction2': jurisdiction2}) 67 | print all_fields 68 | csv_writer.writerow(all_fields[field] for field in task_fields+agency_fields+ ("jurisdiction2",)) 69 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 70 | page += 1 71 | except: 72 | print "OMG!!!!" 73 | #print r 74 | #print r.text 75 | -------------------------------------------------------------------------------- /dogfood/sundayreminder.py: -------------------------------------------------------------------------------- 1 | # -- coding: utf-8 -- 2 | import urllib 3 | import csv 4 | from datetime import datetime, timedelta 5 | import os.path 6 | 7 | 8 | def emojiTranslate(confidence): 9 | print "Confidence is " + str(confidence) 10 | if confidence is "1": 11 | return " 😵" 12 | if confidence is "3": 13 | return " 🙀" 14 | if confidence is "5": 15 | return " 😐" 16 | if confidence is "7": 17 | return " 🙂" 18 | if confidence is "10": 19 | return " 💯" 20 | return "❓" 21 | 22 | 23 | if os.path.isfile('check_in.csv'): 24 | with open('check_in.csv') as csvfile: 25 | print "Inspecting check_in.csv" 26 | mondayNotes = open("sunday_reminder.txt", 'w') 27 | reader = csv.DictReader(csvfile) 28 | mondayNotes.writelines("Attached are last week’s updates for reference for those who used the form; if you didn’t, please forward me along last week’s updates when you get a chance along with this week’s so I can make sure they eventually get added in.\n\n") 29 | mondayNotes.writelines("Check in submission here: https://www.muckrock.com/assignment/monday-check-in-25/ \n") 30 | mondayNotes.writelines("Or copy and paste Google Form\: http://bit.ly/MuckRockStatus \n\n") 31 | mondayNotes.writelines("Quarterly Goals + Last Week's Priorities:\n") 32 | mondayNotes.writelines("==================\n") 33 | for row in reader: 34 | # d = timedelta(row['datetime'],datetime.date.today()) 35 | # if d(days)<5: 36 | if True: 37 | print "working on the update from " + row["user"] 38 | mondayNotes.writelines("\n\n" + row["user"] + "\n") 39 | mondayNotes.writelines("==================\n") 40 | mondayNotes.writelines("Quarterly Goals\n") 41 | mondayNotes.writelines("_______________\n") 42 | mondayNotes.writelines("\n * "+ row["Quarterly Goal #1"]) 43 | mondayNotes.writelines("\n * "+ row["Quarterly Goal #2"]) 44 | mondayNotes.writelines("\n * "+ row["Quarterly Goal #3"]) 45 | 46 | mondayNotes.writelines("\n\nLast Week's Tasks" + "\n") 47 | mondayNotes.writelines("_______________" + "\n") 48 | task = 1 49 | try: 50 | while row["What's this week's prioritized task #" + str(task) + "?"] != "": 51 | mondayNotes.writelines("\n* " + row["What's this week's prioritized task #" + str(task) + "?"]) 52 | task += 1 53 | except: 54 | print "no more tasks" 55 | # misc = ["Upcoming in next four weeks", "Department FYSA", "user"] 56 | # misc = ["user"] 57 | # for section in misc: 58 | # mondayNotes.writelines(section + ":\n") 59 | # mondayNotes.writelines("==================\n") 60 | # for row in reader: 61 | # print "Investigating a row" 62 | # mondayNotes.writelines(row[section] + "__—" + row["user"] + "__\n") 63 | print "All done, saving file." 64 | mondayNotes.close() 65 | else: 66 | print "The check_in.csv is missing." 67 | -------------------------------------------------------------------------------- /export_as_files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import urllib, os, json, datetime, requests, urlparse 5 | import utils 6 | 7 | url = utils.API_URL 8 | token = utils.get_api_key() 9 | headers = utils.get_headers(token) 10 | 11 | # still need to find a nice way to specify pks from console 12 | request_pks = [6996] 13 | 14 | for request_id in request_pks: 15 | print "Working on request " + str(request_id) 16 | # get request first 17 | request_url = url + 'foia/%d/' % request_id 18 | request = requests.get(request_url, headers=headers) 19 | request_data = request.json() 20 | # get agency second 21 | agency_url = url + 'agency/%d/' % request_data['agency'] 22 | agency = requests.get(agency_url , headers=headers) 23 | agency_data = agency.json() 24 | # get communications third 25 | communications = request_data['communications'] 26 | 27 | if communications is None: 28 | print "No communications for request #%d." % request_id 29 | 30 | if not os.path.exists(str(request_id)): # Checks to see if the folder exists. 31 | username = requests.get(url + 'user/%d/' % request_data['user'], headers=headers).json()['username'] 32 | print username 33 | dirName = username + '_' + agency_data['name'] + '_' + request_data['tracking_id'] 34 | # TODO better sanitization on directory names 35 | print "Creating directory " + dirName 36 | dirName = dirName.replace(";", "") # to sanitize it from semi-colons 37 | dirName = dirName.replace(":", "") # to sanitize it from colons 38 | os.makedirs(dirName) 39 | else: 40 | print "The directory already exists. Phew." 41 | 42 | for communication in communications: 43 | commNum = 0 44 | #print communication 45 | for file in communication['files']: 46 | print "Trying to grab a file from communication " + str(commNum) 47 | url = file['ffile'] 48 | split = urlparse.urlsplit(url) # grabbed from http://stackoverflow.com/questions/2795331/python-download-without-supplying-a-filename 49 | filename = split.path.split("/")[-1] 50 | filename = str(communication["date"])+" "+filename 51 | filename = filename.replace(";", "") # to sanitize it from semi-colons 52 | filename = filename.replace(":", "") # to sanitize it from colons 53 | print filename 54 | #urllib.urlretrieve(url, '/'+str(request_id)+'/'+filename) 55 | urllib.urlretrieve(url, dirName + '/' + filename) 56 | 57 | 58 | print "Trying to grab the text from the communication" 59 | # eventually this should save to pdf, maybe using this: https://github.com/mstamy2/PyPDF2/tree/master/Sample_Code 60 | 61 | communicationText = communication["communication"].encode('ascii', 'ignore') 62 | text_file = open(dirName + '/' + communication["date"] + " Communication.txt", "w+") 63 | text_file.write(communicationText) 64 | text_file.close() 65 | -------------------------------------------------------------------------------- /dogfood/mondayreport.py: -------------------------------------------------------------------------------- 1 | # -- coding: utf-8 -- 2 | import urllib 3 | import csv 4 | from datetime import datetime, timedelta 5 | import os.path 6 | 7 | print "Getting ready to save Michael 15 minutes or more." 8 | 9 | def emojiTranslate(confidence): 10 | print "Confidence is " + str(confidence) 11 | if confidence is "1": 12 | return " 😵" 13 | if confidence is "3": 14 | return " 🙀" 15 | if confidence is "5": 16 | return " 😐" 17 | if confidence is "7": 18 | return " 🙂" 19 | if confidence is "10": 20 | return " 💯" 21 | return "❓" 22 | 23 | 24 | if os.path.isfile('check_in.csv'): 25 | with open('check_in.csv') as csvfile: 26 | print "Inspecting check_in.csv" 27 | mondayNotes = open("mondaynotes.txt", 'w') 28 | print "Opening up the Monday notes." 29 | reader = csv.DictReader(csvfile) 30 | mondayNotes.writelines("New Goals + Last Week's Priorities:\n") 31 | mondayNotes.writelines("==================\n") 32 | mondayNotes.writelines("\n") 33 | 34 | for row in reader: 35 | # d = timedelta(row['datetime'],datetime.date.today()) 36 | # if d(days)<5: 37 | if True: 38 | print "working on the update from " + row["user"] 39 | mondayNotes.writelines("\n\n" + row["user"] + "\n") 40 | mondayNotes.writelines("==================\n") 41 | mondayNotes.writelines("Quarterly Goals\n") 42 | mondayNotes.writelines("_______________\n") 43 | mondayNotes.writelines(row["Quarterly Goal #1"]) 44 | mondayNotes.writelines(emojiTranslate(row["Goal #1 Confidence"]) + "\n") 45 | mondayNotes.writelines(row["Quarterly Goal #2"]) 46 | mondayNotes.writelines(emojiTranslate(row["Goal #2 Confidence"]) + "\n") 47 | mondayNotes.writelines(row["Quarterly Goal #3"]) 48 | mondayNotes.writelines(emojiTranslate(row["Goal #3 Confidence"]) + "\n") 49 | mondayNotes.writelines("\n\nLast Week's Tasks" + "\n") 50 | mondayNotes.writelines("_______________" + "\n") 51 | task = 1 52 | try: 53 | while row["Last week's task #" + str(task)] != "": 54 | mondayNotes.writelines("* " + row["Last week's task #" + str(task)] + ": " + row["Was task #" + str(task) + " achieved? Anything we need to adjust?"] + "\n") 55 | task += 1 56 | except: 57 | print "no more tasks" 58 | mondayNotes.writelines("\n\nThis Week's Tasks" + "\n") 59 | mondayNotes.writelines("_______________" + "\n") 60 | task = 1 61 | try: 62 | while row["What's this week's prioritized task #" + str(task) + "?"] != "": 63 | mondayNotes.writelines("* " + row["What's this week's prioritized task #" + str(task) + "?"]) 64 | task += 1 65 | except: 66 | print "no more tasks" 67 | # misc = ["Upcoming in next four weeks", "Department FYSA", "user"] 68 | # misc = ["user"] 69 | # for section in misc: 70 | # mondayNotes.writelines(section + ":\n") 71 | # mondayNotes.writelines("==================\n") 72 | # for row in reader: 73 | # print "Investigating a row" 74 | # mondayNotes.writelines(row[section] + "__—" + row["user"] + "__\n") 75 | print "All done, saving file." 76 | mondayNotes.close() 77 | else: 78 | print "The check_in.csv is missing." 79 | -------------------------------------------------------------------------------- /bulk_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import json 6 | import csv 7 | 8 | # Step 1: Create a file "file_with_agencies.csv" and add the agency IDs you want to file with in it, one per line. You can use 248 and 21983 as test agencies. You do not normally need to modify the following line. Filing with test agencies will deduct requests from your account, but if you reach out we can refund those requests or you can cancel them yourself within 30 minutes. 9 | 10 | 11 | # Step 2: Create a file "requestLanguage.txt" with the text of the request. This should be the full length of the request and include any salutations. Do not include contact information — MuckRock generates this on a per request basis. Update the below with your request's title as indicated 12 | 13 | f = open("requestLanguage.txt","r") 14 | 15 | # Step 3: When possible, we recommend avoiding adding attachments as agencies receive requests in a variety of ways. If needed, you can define a file to attach here. Uncomment this line, update the attachment name as appropriate, and then uncomment out the line beginning ""# 'attachments':" below as well. 16 | 17 | # attachment = "file.pdf" 18 | 19 | # Step 3: Check your API key from your profile page, located lower left of the page here: https://www.muckrock.com/accounts/profile/ Put that API key below, as indicated: 20 | 21 | token = "1234" 22 | 23 | # Be careful: Anyone with access to your API key has full access to your account! 24 | 25 | # Step 4: Uncomment embargo settings ("'embargo': True") as appropriate. Using embargo is dependent on your plan type on MuckRock. 26 | 27 | # Step 5: Execute command, python2 bulk_file.py 28 | 29 | 30 | with open('file_with_agencies.csv', 'rt') as f: 31 | reader = csv.reader(f) 32 | agencies = list(reader) 33 | 34 | url = 'https://www.muckrock.com/api_v1/' 35 | headers = { 36 | 'Authorization': 'Token %s' % token, 37 | 'content-type': 'application/json' 38 | } 39 | 40 | print("About to file this request .... /n") 41 | f = open("requestLanguage.txt","rt") 42 | message = f.read() 43 | 44 | print(message) 45 | 46 | for agency in agencies: 47 | 48 | # Sample Structure: https://www.muckrock.com/api_v1/agency/100/ 49 | agency_url = url + 'agency/' + str(agency[0]) + '/' 50 | print(agency_url) 51 | a = requests.get(agency_url, headers=headers) 52 | agency_name = a.json()['name'] 53 | title = agency_name + " Request" # Update this with title 54 | data = json.dumps({ 55 | 'agency': agency[0], 56 | 'title': title, 57 | 'full_text': message, 58 | # 'attachments': [attachment], 59 | # 'embargo': True 60 | 'permanent_embargo': True 61 | }) 62 | 63 | # The request will be saved as a draft if you do not have any requests left 64 | r = requests.post(url + 'foia/', headers=headers, data=data) 65 | print(r) 66 | print("If it says 201, succesfully filed with " + agency_name) 67 | # 402 errors indicate that your request did not have enouhg requests to file, and saved it as a draft. 68 | # You may need to log in to MuckRock and switch the active organization to make sure you're drawing from the correct pool. 69 | -------------------------------------------------------------------------------- /export_agency_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | from time import sleep 4 | 5 | import requests 6 | import unicodecsv 7 | 8 | from utils import get_api_key 9 | 10 | token = get_api_key() 11 | url = 'https://www.muckrock.com/api_v1/' 12 | 13 | headers = {'Authorization': 'Token %s' % token, 'content-type': 'application/json'} 14 | next_ = url + 'agency' 15 | 16 | fields = ( 17 | "id", 18 | "name", 19 | "slug", 20 | "status", 21 | "twitter", 22 | "twitter_handles", 23 | "parent", 24 | "appeal_agency", 25 | "url", 26 | "foia_logs", 27 | "foia_guide", 28 | "public_notes", 29 | "absolute_url", 30 | "average_response_time", 31 | "fee_rate", 32 | "success_rate", 33 | "has_portal", 34 | "has_email", 35 | "has_fax", 36 | "has_address", 37 | "number_requests", 38 | "number_requests_completed", 39 | "number_requests_rejected", 40 | "number_requests_no_docs", 41 | "number_requests_ack", 42 | "number_requests_resp", 43 | "number_requests_fix", 44 | "number_requests_appeal", 45 | "number_requests_pay", 46 | "number_requests_partial", 47 | "number_requests_lawsuit", 48 | "number_requests_withdrawn" 49 | ) 50 | jurisdiction_fields = ( 51 | 'name', 52 | 'parent', 53 | 'level', 54 | ) 55 | 56 | page = 1 57 | 58 | # make this true while exporting data to not crash on errors 59 | SUPRESS_ERRORS = False 60 | 61 | # This allows you to cach jurisdiction look ups 62 | jurisdictions = {} 63 | 64 | def get_jurisdiction(jurisdiction_id): 65 | global jurisdictions 66 | if jurisdiction_id in jurisdictions: 67 | return jurisdictions[jurisdiction_id] 68 | else: 69 | # print 'getting jurisdiction', jurisdiction_id 70 | sleep(1) # rate limit 71 | r = requests.get(url + 'jurisdiction/' + str(jurisdiction_id), headers=headers) 72 | jurisdiction_json = r.json() 73 | if jurisdiction_json['parent']: # USA has no paremt 74 | parent = get_jurisdiction(jurisdiction_json['parent']) 75 | jurisdiction_json['parent'] = parent['name'] # replace parent id with parent name in jurisdiction json 76 | jurisdictions[jurisdiction_id] = jurisdiction_json 77 | return jurisdiction_json 78 | 79 | csv_file = open('agency_stats.csv', 'w') 80 | csv_writer = unicodecsv.writer(csv_file) 81 | jurisdiction_field_names = tuple('jurisdiction {}'.format(f) for f in jurisdiction_fields) 82 | csv_writer.writerow(fields + jurisdiction_field_names) 83 | 84 | while next_ is not None: 85 | r = requests.get(next_, headers=headers) 86 | try: 87 | json = r.json() 88 | next_ = json['next'] 89 | for datum in json['results']: 90 | agency_values = [datum[field] for field in fields] 91 | jurisdiction = get_jurisdiction(datum['jurisdiction']) 92 | jurisdiction_values = [jurisdiction[field] for field in jurisdiction_fields] 93 | csv_writer.writerow(agency_values + jurisdiction_values) 94 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 95 | page += 1 96 | except Exception as e: 97 | print 'Error', e 98 | if not SUPRESS_ERRORS: 99 | raise 100 | -------------------------------------------------------------------------------- /export_all_requests_as_files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import urllib, os, json, datetime, requests, urlparse 5 | import utils, time 6 | 7 | url = utils.API_URL 8 | token = utils.get_api_key() 9 | headers = utils.get_headers(token) 10 | 11 | 12 | 13 | page = 1 14 | next_ = url+"foia/" 15 | 16 | 17 | 18 | 19 | while next_ is not None: # Handling at the page level 20 | r = requests.get(next_, headers=headers) 21 | try: 22 | json_data = r.json() 23 | print 'Page %d of %d' % (page, json_data['count'] / 20 + 1) 24 | next_ = json_data['next'] 25 | for request in json_data['results']: 26 | print "Working on request " + str(request["id"]) 27 | request_url = url + 'foia/%d/' % request["id"] 28 | r2 = requests.get(request_url, headers=headers) #original 29 | 30 | request_data = r2.json() 31 | 32 | print "Here is the agency number " + str(request_data["agency"]) 33 | agency = requests.get("https://www.muckrock.com/api_v1/agency/" + str(request_data["agency"]), headers=headers) 34 | 35 | agency_data = agency.json() 36 | # get communications third 37 | 38 | communications = request_data['communications'] 39 | 40 | if communications is None: 41 | print "No communications for request #%d." % request_data["id"] 42 | 43 | if not os.path.exists(str(request_data["id"])): # Checks to see if the folder exists. 44 | username = requests.get(url + 'user/%d/' % request_data['user'], headers=headers).json()['username'] 45 | print username 46 | dirName = username + '_' + agency_data['name'] + '_' + request_data['tracking_id'] 47 | # TODO better sanitization on directory names 48 | print "Creating directory " + dirName 49 | dirName = dirName.replace(";", "") # to sanitize it from semi-colons 50 | dirName = dirName.replace(":", "") # to sanitize it from colons 51 | os.makedirs(dirName) 52 | else: 53 | print "The directory already exists. Phew." 54 | 55 | for communication in communications: 56 | #print communication 57 | for file in communication['files']: 58 | fileurl = file['ffile'] 59 | split = urlparse.urlsplit(fileurl) # grabbed from http://stackoverflow.com/questions/2795331/python-download-without-supplying-a-filename 60 | filename = split.path.split("/")[-1] 61 | filename = str(communication["date"])+" "+filename 62 | filename = filename.replace(";", "") # to sanitize it from semi-colons 63 | filename = filename.replace(":", "") # to sanitize it from colons 64 | urllib.urlretrieve(fileurl, dirName + '/' + filename) 65 | 66 | 67 | print "Trying to grab the text from the communication" 68 | # eventually this should save to pdf, maybe using this: https://github.com/mstamy2/PyPDF2/tree/master/Sample_Code 69 | 70 | communicationText = communication["communication"].encode('ascii', 'ignore') 71 | text_file = open(dirName + '/' + communication["date"] + " Communication.txt", "w+") 72 | text_file.write(communicationText) 73 | text_file.close() 74 | print "File closed" 75 | time.sleep(5) 76 | 77 | except: 78 | print "There was an error of unkown origin" 79 | # print r 80 | # print r.text 81 | -------------------------------------------------------------------------------- /export_standard_full.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | # Standard Library 5 | import datetime 6 | import json 7 | import os 8 | import time 9 | 10 | # Third Party 11 | import requests 12 | from simplejson.scanner import JSONDecodeError 13 | 14 | import utils 15 | 16 | url = utils.API_URL 17 | token = utils.get_api_key() 18 | headers = utils.get_headers(token) 19 | 20 | page = 599 21 | next_ = url + 'foia/?embargo=3&page=%d' % page 22 | 23 | agencies = {} 24 | jurisdictions = {} 25 | 26 | def get_jurisdiction(jurisdiction_id): 27 | global jurisdictions 28 | if jurisdiction_id in jurisdictions: 29 | return jurisdictions[jurisdiction_id] 30 | else: 31 | print 'getting jurisdiction', jurisdiction_id 32 | r = requests.get(url + 'jurisdiction/' + str(jurisdiction_id), headers=headers) 33 | jurisdiction_json = r.json() 34 | jurisdiction = '%s_%s' % (jurisdiction_id, jurisdiction_json['slug']) 35 | jurisdictions[jurisdiction_id] = jurisdiction 36 | return jurisdiction 37 | 38 | 39 | while next_ is not None: # Handling at the page level 40 | try: 41 | r = requests.get(next_, headers=headers) 42 | json_data = r.json() 43 | print 'Page %d of %d (%d total)' % (page, json_data['count'] / 50 + 1, json_data['count']) 44 | page += 1 45 | next_ = json_data['next'] 46 | print next_ 47 | for request in json_data['results']: 48 | print 'Working on request ' + str(request['id']) 49 | 50 | if request['status'] == 'started': 51 | continue 52 | 53 | if request['agency'] in agencies: 54 | agency, jurisdiction = agencies[request['agency']] 55 | else: 56 | if request['agency'] is None: 57 | agency = 'None' 58 | jurisdiction = 'None' 59 | else: 60 | print 'getting agency', request['agency'] 61 | r = requests.get(url + 'agency/' + str(request['agency']), headers=headers) 62 | agency_json = r.json() 63 | agency = '%s_%s' % (request['agency'], agency_json['slug']) 64 | jurisdiction = get_jurisdiction(agency_json['jurisdiction']) 65 | agencies[request['agency']] = (agency, jurisdiction) 66 | 67 | communications = request['communications'] 68 | 69 | dir_name = '%s/%s/%s_%s' % (jurisdiction, agency, request['id'], request['slug']) 70 | if not os.path.exists(dir_name): 71 | os.makedirs(dir_name) 72 | 73 | for i, communication in enumerate(communications): 74 | for file_ in communication['files']: 75 | fileurl = file_['ffile'] 76 | file_name = '%s_%s' % (file_['datetime'], fileurl.split('/')[-1]) 77 | file_name = '%s/%s' % (dir_name, file_name) 78 | if not os.path.exists(file_name): 79 | with open(file_name, 'wb') as f: 80 | f.write(requests.get(fileurl).content) 81 | 82 | communication_text = communication['communication'].encode('ascii', 'ignore') 83 | date = communication['datetime'].split('T')[0] 84 | with open('%s/%s_%s_communication.txt' % (dir_name, date, i), 'w') as f: 85 | f.write(communication_text) 86 | 87 | except JSONDecodeError: 88 | print r.status_code 89 | print r.content 90 | raise 91 | except KeyboardInterrupt: 92 | import ipdb 93 | ipdb.set_trace() 94 | 95 | -------------------------------------------------------------------------------- /export_all_user_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import utils 5 | import urllib, os, json, datetime, requests, urlparse 6 | 7 | 8 | api_url = utils.API_URL 9 | token = utils.get_api_key() 10 | headers = utils.get_headers(token) 11 | 12 | username = raw_input('Username: ') 13 | next_url = api_url + "foia/?user=" + username 14 | current_page = 0 15 | 16 | while next_url: 17 | # we use next_url because the API results are paginated 18 | r = requests.get(next_url, headers=headers) 19 | data = r.json() 20 | next_url = data['next'] 21 | 22 | # measures progress by page, not by result 23 | current_page += 1 24 | total_pages = (data['count'] / 20.0) 25 | utils.display_progress(current_page, total_pages) 26 | 27 | for result in data['results']: 28 | request_id = result['id'] 29 | print "Working on request " + str(request_id) 30 | # get request first 31 | request_url = api_url + 'foia/%d/' % request_id 32 | print request_url 33 | request = requests.get(request_url, headers=headers) 34 | print request 35 | request_data = request.json() 36 | # get agency second 37 | agency_url = api_url + 'agency/%d/' % request_data['agency'] 38 | agency = requests.get(agency_url , headers=headers) 39 | agency_data = agency.json() 40 | # get communications third 41 | communications = request_data['communications'] 42 | 43 | if communications is None: 44 | print "No communications for request #%d." % request_id 45 | 46 | if not os.path.exists(str(request_id)): # Checks to see if the folder exists. 47 | username = requests.get(api_url + 'user/%d/' % request_data['user'], headers=headers).json()['username'] 48 | print username 49 | dirName = username + '_' + agency_data['name'] + '_' + str(request_data['id']) 50 | # TODO better sanitization on directory names 51 | print "Creating directory " + dirName 52 | dirName = dirName.replace(";", "") # to sanitize it from semi-colons 53 | dirName = dirName.replace(":", "") # to sanitize it from colons 54 | os.makedirs(dirName) 55 | else: 56 | print "The directory already exists. Phew." 57 | 58 | for communication in communications: 59 | commNum = 0 60 | #print communication 61 | for file in communication['files']: 62 | print "Trying to grab a file from communication " + str(commNum) 63 | url = file['ffile'] 64 | split = urlparse.urlsplit(url) # grabbed from http://stackoverflow.com/questions/2795331/python-download-without-supplying-a-filename 65 | filename = split.path.split("/")[-1] 66 | filename = str(communication["date"])+" "+filename 67 | filename = filename.replace(";", "") # to sanitize it from semi-colons 68 | filename = filename.replace(":", "") # to sanitize it from colons 69 | print filename 70 | #urllib.urlretrieve(url, '/'+str(request_id)+'/'+filename) 71 | urllib.urlretrieve(url, dirName + '/' + filename) 72 | 73 | 74 | print "Trying to grab the text from the communication" 75 | # eventually this should save to pdf, maybe using this: https://github.com/mstamy2/PyPDF2/tree/master/Sample_Code 76 | 77 | communicationText = communication["communication"].encode('ascii', 'ignore') 78 | text_file = open(dirName + '/' + communication["date"] + " Communication.txt", "w+") 79 | text_file.write(communicationText) 80 | text_file.close() 81 | -------------------------------------------------------------------------------- /export_users_requests_as_files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import datetime 5 | import json 6 | import os 7 | import time 8 | import urllib 9 | import urlparse 10 | 11 | import requests 12 | 13 | import utils 14 | 15 | url = utils.API_URL 16 | token = utils.get_api_key() 17 | headers = utils.get_headers(token) 18 | 19 | 20 | user = raw_input('Username to export (case sensitive):') 21 | # user = "USERNAME" #<--- Put username here. Case sensitive. 22 | # request_pks = [6996] #<--- To export by MR number 23 | 24 | page = 1 25 | next_ = url+"foia/?user="+user 26 | 27 | 28 | 29 | 30 | while next_ is not None: # Handling at the page level 31 | resp = requests.get(next_, headers=headers) 32 | try: 33 | json_data = resp.json() 34 | print "\n" + ("*" * 100) 35 | print 'Page %d of %d' % (page, json_data['count'] / 50 + 1) 36 | print ("*" * 100) + "\n" 37 | page += 1 38 | next_ = json_data['next'] 39 | for request in json_data['results']: 40 | time.sleep(3) 41 | print "\nWorking on request " + str(request["id"]) 42 | request_url = url + 'foia/%d/' % request["id"] 43 | resp = requests.get(request_url, headers=headers) #original 44 | 45 | request_data = resp.json() 46 | 47 | print "Here is the agency number " + str(request_data["agency"]) 48 | resp = requests.get("https://www.muckrock.com/api_v1/agency/" + str(request_data["agency"]), headers=headers) 49 | 50 | agency_data = resp.json() 51 | # get communications third 52 | 53 | communications = request_data['communications'] 54 | 55 | if communications is None: 56 | print "No communications for request #%d." % request_data["id"] 57 | 58 | resp = requests.get(url + 'user/%d/' % request_data['user'], headers=headers) 59 | username = resp.json()['username'] 60 | print username 61 | dirName = username + '_' + agency_data['name'] + '_' + str(request_data['id']) 62 | if not os.path.exists(dirName): # Checks to see if the folder exists. 63 | # TODO better sanitization on directory names 64 | print "Creating directory " + dirName 65 | dirName = dirName.replace(";", "") # to sanitize it from semi-colons 66 | dirName = dirName.replace(":", "") # to sanitize it from colons 67 | os.makedirs(dirName) 68 | else: 69 | print "The directory already exists. Phew.", dirName 70 | 71 | with open(dirName + "/agency.txt", "w") as agency_file: 72 | for email in agency_data["emails"]: 73 | if email["request_type"] == "primary" and email["email_type"] == "to": 74 | agency_file.write(email["email"]["email"] + "\n\n") 75 | break 76 | for addr in agency_data["addresses"]: 77 | if addr["request_type"] == "primary": 78 | agency_file.write("%(street)s, %(city)s %(state)s %(zip_code)s\n\n" % addr["address"]) 79 | break 80 | 81 | for communication in communications: 82 | #print communication 83 | for file in communication['files']: 84 | fileurl = file['ffile'] 85 | split = urlparse.urlsplit(fileurl) # grabbed from http://stackoverflow.com/questions/2795331/python-download-without-supplying-a-filename 86 | filename = split.path.split("/")[-1] 87 | filename = str(communication["datetime"])+" "+filename 88 | filename = filename.replace(";", "") # to sanitize it from semi-colons 89 | filename = filename.replace(":", "") # to sanitize it from colons 90 | urllib.urlretrieve(fileurl, dirName + '/' + filename) 91 | 92 | 93 | print "Trying to grab the text from the communication" 94 | # eventually this should save to pdf, maybe using this: https://github.com/mstamy2/PyPDF2/tree/master/Sample_Code 95 | 96 | communicationText = communication["communication"].encode('ascii', 'ignore') 97 | text_file = open(dirName + '/' + communication["datetime"] + " Communication.txt", "w+") 98 | text_file.write(communicationText) 99 | text_file.close() 100 | print "File closed" 101 | 102 | except Exception as exc: 103 | print "There was an error of unkown origin" 104 | print(resp.content) 105 | raise 106 | -------------------------------------------------------------------------------- /summarize_open_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests as http_request 5 | import json 6 | import utils 7 | import unicodecsv 8 | 9 | 10 | unacknowledged_file = open('cialetter_unacknowledged.txt', 'w') 11 | acknowledged_file = open('cialetter_acknowledged.txt', 'w') 12 | 13 | url = utils.API_URL 14 | token = utils.get_api_key() 15 | headers = utils.get_headers(token) 16 | 17 | agencyID = raw_input('AgencyID: ') 18 | next_url = "https://www.muckrock.com/api_v1/foia/?agency=" + agencyID 19 | current_page = 0 20 | 21 | while next_url: 22 | # we use next_url because the API results are paginated 23 | r = http_request.get(next_url, headers=headers) 24 | data = r.json() 25 | next_url = data['next'] 26 | 27 | # measures progress by page, not by result 28 | current_page += 1 29 | total_pages = (data['count'] / 20.0) 30 | utils.display_progress(current_page, total_pages) 31 | unacknowledged_file.write("Below are requests that we have no record of ever being acknowledged. We have included the first 200 characters of the request, as well as subject heading, for help in identifying the request.\n\n\n") 32 | acknowledged_file.write("Below are requests that were acknolwedged but we have not yet recevied a final response for. We have included the first 200 characters of the request, as well as subject heading, for help in identifying the request. \n\n\n") 33 | 34 | 35 | for request in data['results']: 36 | if request['status'] == 'ack': 37 | # Means Awaitng Ack 38 | request_id = request['id'] 39 | request_url = 'https://www.muckrock.com/api_v1/foia/%s/' % str(request_id) 40 | 41 | print "Working on request " + str(request_id) 42 | 43 | 44 | unacknowledged_file.write("\n" + "\n"+ "Subject Heading: " + str(request['title'].encode('utf-8').strip()) + "\n" + "\n") 45 | if request['embargo'] is False: 46 | unacknowledged_file.write("Request URL: muckrock.com" + str(request['absolute_url']) + "\n" + "\n") 47 | unacknowledged_file.write("Summary: " + str(request['description'].encode('utf-8').strip())[0:300] + "\n" + "\n") 48 | unacknowledged_file.write("Submitted: " + str(request['date_submitted']) + "\n") 49 | if request['communications'][0]["from_who"]: 50 | print "Got a requester's name! It's " + request['communications'][0]["from_who"] 51 | unacknowledged_file.write("Requester Name: " + request['communications'][0]["from_who"] + "\n") 52 | else: 53 | print "No requester's name" 54 | 55 | unacknowledged_file.write("Address associated with this request:\n\nMuckRock \n") 56 | unacknowledged_file.write("DEPT MR" + str(request_id)) 57 | unacknowledged_file.write("411A Highland Ave\nSomerville, MA 02144-2516") 58 | unacknowledged_file.write("\n\n\n================================\n================================\n================================\n") 59 | 60 | 61 | elif request['status'] == 'processed': 62 | #processed is Awaiting Response 63 | request_id = request['id'] 64 | request_url = 'https://www.muckrock.com/api_v1/foia/%s/' % str(request_id) 65 | acknowledged_file.write("\n" + "\n"+ "Subject Heading: " + str(request['title'].encode('utf-8').strip()) + "\n" + "\n") 66 | 67 | if request['embargo'] is False: 68 | acknowledged_file.write("Request URL: muckrock.com" + str(request['absolute_url']) + "\n" + "\n") 69 | 70 | acknowledged_file.write("Summary: " + str(request['description'].encode('utf-8').strip())[0:300] + "\n" + "\n") 71 | acknowledged_file.write("Submitted: " + str(request['date_submitted']) + "\n") 72 | 73 | if request['communications'][0]["from_who"]: 74 | print "Got a requester's name! It's " + request['communications'][0]["from_who"] 75 | acknowledged_file.write("Requester Name: " + request['communications'][0]["from_who"] + "\n") 76 | else: 77 | print "No requesters name" 78 | if request['date_due'] is not None: 79 | acknowledged_file.write("Estimated Completion Date: " + str(request['date_due'])+ "\n\n") 80 | else: 81 | acknowledged_file.write("Estimated Completion Date: We have not received an estimated completion date for this request.\n\n") 82 | if request['tracking_id'] is not None: 83 | acknowledged_file.write("Tracking Number: " + str(request['tracking_id'])+ "\n\n") 84 | else: 85 | acknowledged_file.write("Tracking Number: We have not received a tracking number for this request.\n\n") 86 | acknowledged_file.write("Last Response From Agency: " + str(request['date_followup'])+ "\n" + "\n") 87 | acknowledged_file.write("Address associated with this request:\n\nMuckRock \n") 88 | acknowledged_file.write("DEPT MR" + str(request_id)) 89 | acknowledged_file.write("411A Highland Ave\nSomerville, MA 02144-2516") 90 | acknowledged_file.write("\n\n\n================================\n================================\n================================\n") 91 | 92 | else: #put stuff here 93 | print "No Match." 94 | -------------------------------------------------------------------------------- /export_foia.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import requests 5 | import unicodecsv 6 | import utils 7 | import datetime 8 | import time 9 | from utils import get_api_key 10 | 11 | url = utils.API_URL 12 | 13 | token = get_api_key() 14 | 15 | url = 'https://www.muckrock.com/api_v1/' 16 | headers = utils.get_headers(token) 17 | 18 | file_object = open('log.txt', 'a') 19 | 20 | 21 | fields = ( 22 | 'id', 23 | 'title', 24 | 'agency_id', 25 | 'agency', 26 | 'jurisdiction_id', 27 | 'jurisdiction', 28 | 'level', 29 | 'parent', 30 | 'absolute_url', 31 | 'status', 32 | 'user', 33 | "embargo", 34 | "permanent_embargo", 35 | "datetime_submitted", 36 | "date_due", 37 | "days_until_due", 38 | "date_followup", 39 | "datetime_done", 40 | "date_embargo", 41 | "tracking_id", 42 | "price", 43 | "disable_autofollowups", 44 | "tags", 45 | "notes" 46 | 47 | ) 48 | 49 | #user = raw_input('Username to export (case sensitive):') 50 | #user = "Morisy" #<--- Put username here. Case sensitive. 51 | # request_pks = [6996] #<--- To export by MR number 52 | 53 | page = 1 # Defined up here to ease rusuming. Should start at 1 unless resuming 54 | #next_ = url+"foia/?user="+user #<- use this one if you want to limit to a particular user's requests 55 | next_ = url+"foia/?page="+str(page) # <- use this one if you want to start from a page after a crash or failure 56 | 57 | 58 | csv_file = open('foia_data_' + str(datetime.date.today()) + '.csv', 'w') 59 | csv_file.seek(0) 60 | csv_writer = unicodecsv.writer(csv_file) 61 | csv_writer.writerow(fields) 62 | 63 | while next_ is not None: 64 | r = requests.get(next_, headers=headers) 65 | try: 66 | json = r.json() 67 | next_ = json['next'] 68 | for datum in json['results']: 69 | try: 70 | items = [] 71 | print "Working on request with ID " + str(datum['id']) 72 | for field in fields: 73 | if field == 'jurisdiction' or field == 'jurisdiction_id' or field == 'level' or field == 'parent' or field == "agency_id": 74 | four = 4 # I just need something on this line 75 | elif field == 'agency': 76 | items.append(datum['agency']) 77 | agency_url = "https://www.muckrock.com/api_v1/agency/" + str(datum['agency']) + '/' 78 | agency = requests.get(agency_url , headers=headers) 79 | agency_data = agency.json() 80 | agency_name = agency_data['name'] 81 | items.append(agency_name) ## Things work through here 82 | items.append(str(agency_data["jurisdiction"])) 83 | jurisdiction_url = "https://www.muckrock.com/api_v1/jurisdiction/" + str(agency_data["jurisdiction"]) + '/' 84 | jurisdiction = requests.get(jurisdiction_url , headers=headers) 85 | jurisdiction_data = jurisdiction.json() 86 | items.append(jurisdiction_data['name']) ## Things work through here 87 | items.append(jurisdiction_data['level']) 88 | if jurisdiction_data['parent'] != None: 89 | if jurisdiction_data['level'] == 's': 90 | items.append(jurisdiction_data['name']) 91 | else: 92 | jurisdiction_url = "https://www.muckrock.com/api_v1/jurisdiction/" + str(jurisdiction_data['parent']) + '/' 93 | jurisdiction = requests.get(jurisdiction_url , headers=headers) 94 | jurisdiction_data = jurisdiction.json() 95 | items.append(jurisdiction_data['name']) 96 | else: 97 | items.append("United States of America") 98 | elif field == 'tracking_id': 99 | if datum['tracking_id'] == "": 100 | items.append("No") 101 | else: 102 | items.append("Yes") 103 | elif field == "notes": 104 | all_notes = "" 105 | if datum['notes'] == "": 106 | items.append("") 107 | else: 108 | for note in datum['notes']: 109 | all_notes += " " + note["note"] 110 | items.append(all_notes) 111 | else: 112 | items.append(datum[field]) 113 | csv_writer.writerow(items) 114 | except Exception as e: 115 | file_object.write('\nError Type 1: There was an error on MR' + str(datum['id']) + ' page ' + str(page)) 116 | print 'There was an error on MR' + str(datum['id']) + 'page ' + str(page) 117 | print e 118 | time.sleep(3) 119 | # print r.content 120 | print 'Page %d of %d' % (page, (json['count'] / 20 )) 121 | page += 1 122 | except Exception as e: 123 | file_object.write('\nError Type 2: There was an error on page ' + str(page)) 124 | print 'There was an error on page ' + str(page) 125 | print e 126 | # print r.content 127 | time.sleep(3) 128 | -------------------------------------------------------------------------------- /dogfood/monday_reloaded.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -- coding: utf-8 -- 3 | 4 | import urllib 5 | import unicodecsv as csv 6 | from datetime import timedelta 7 | import datetime 8 | import os.path 9 | import requests 10 | import json 11 | import markdown 12 | import matplotlib.pyplot as plt 13 | import numpy as np 14 | import pandas as pd 15 | from mailchimp3 import MailChimp 16 | import utils 17 | from utils import get_api_key 18 | from dateutil import parser 19 | 20 | 21 | import sys 22 | reload(sys) 23 | sys.setdefaultencoding('utf8') 24 | 25 | url = utils.API_URL 26 | token = utils.get_api_key() 27 | headers = utils.get_headers(token) 28 | 29 | mondayNotes = open("mondaynotes.md", 'w') 30 | 31 | mailChimpKey = raw_input('What is your Mailchimp API Key? ') 32 | 33 | token = get_api_key() 34 | 35 | url = 'https://www.muckrock.com/api_v1/' 36 | headers = {'Authorization': 'Token %s' % token, 'content-type': 'application/json'} 37 | 38 | page = 1 39 | days = 7 40 | 41 | staff_names = ( 42 | "JPat Brown", 43 | "Caitlin Russell", 44 | "Beryl Lipton", 45 | "Jessie Gomez", 46 | "Mitchell Kotler", 47 | "Dylan Freedman", 48 | "Aron Pilhofer", 49 | "Michael Morisy" 50 | ) 51 | 52 | class staffer_log: 53 | goal1 = "" 54 | goal2 = "" 55 | goal3 = "" 56 | goal1data = [] 57 | goal2data = [] 58 | goal3data = [] 59 | tasks = [] 60 | previousTasks = [] 61 | 62 | def getWeeklyUpdates(staffer): 63 | print "Working on the goals progress the past week for " + staffer 64 | next_ = url + 'assignment-responses/?crowdsource=25&ordering=-id' 65 | day = 0 66 | page = 1 67 | submitter = staffer_log() 68 | submitter.tasks = [] 69 | submitter.previousTasks = [] 70 | while day < 1 and next_ is not None: 71 | r = requests.get(next_, headers=headers) 72 | try: 73 | json = r.json() 74 | next_ = json['next'] 75 | for datum in json['results']: 76 | if datum["user"] == staffer: 77 | if day == 0: 78 | submitter.goal1 = [datum][0]['values'][0]['value'] 79 | submitter.goal2 = [datum][0]['values'][2]['value'] 80 | submitter.goal3 = [datum][0]['values'][4]['value'] 81 | submitter.goal1data = [datum["values"][1]['value']] 82 | submitter.goal2data = [datum["values"][3]['value']] 83 | submitter.goal3data = [datum["values"][5]['value']] 84 | if datum["values"][16]['value']: 85 | submitter.tasks.append(datum["values"][16]['value']) 86 | if datum["values"][17]['value']: 87 | submitter.tasks.append(datum["values"][17]['value']) 88 | if datum["values"][18]['value']: 89 | submitter.tasks.append(datum["values"][18]['value']) 90 | if datum["values"][19]['value']: 91 | submitter.tasks.append(datum["values"][19]['value']) 92 | if datum["values"][20]['value']: 93 | submitter.tasks.append(datum["values"][20]['value']) 94 | if datum["values"][6]['value']: 95 | submitter.previousTasks.append(str(datum["values"][6]['value'] + ": " + datum["values"][7]['value'])) 96 | if datum["values"][8]['value']: 97 | submitter.previousTasks.append(str(datum["values"][8]['value'] + ": " + datum["values"][9]['value'])) 98 | if datum["values"][10]['value']: 99 | submitter.previousTasks.append(str(datum["values"][10]['value'] + ": " + datum["values"][11]['value'])) 100 | if datum["values"][12]['value']: 101 | submitter.previousTasks.append(str(datum["values"][12]['value'] + ": " + datum["values"][13]['value'])) 102 | if datum["values"][14]['value']: 103 | submitter.previousTasks.append(str(datum["values"][14]['value'] + ": " + datum["values"][15]['value'])) 104 | else: 105 | submitter.goal1data.insert(0, datum["values"][1]['value']) 106 | submitter.goal2data.insert(0, datum["values"][3]['value']) 107 | submitter.goal3data.insert(0, datum["values"][5]['value']) 108 | day += 1 109 | print 'Page %d of %d' % (page, json['count'] / 10 + 1) 110 | page += 1 111 | except Exception as e: 112 | print e 113 | 114 | return submitter 115 | 116 | def getAssignmentStats(): 117 | fields = ( 118 | "id", 119 | "crowdsource", 120 | "user", 121 | "edit_user", 122 | "data", 123 | "datetime", 124 | "edit_datetime", 125 | "tags", 126 | "skip", 127 | "number", 128 | "flag", 129 | "gallery" 130 | ) 131 | page = 1 132 | 133 | 134 | csv_file = open('assignment_responses.csv', 'w') 135 | csv_file.seek(0) 136 | csv_writer = csv.writer(csv_file) 137 | csv_writer.writerow(fields) 138 | next_ = url + 'assignment-responses' 139 | while next_ is not None: 140 | r = requests.get(next_, headers=headers) 141 | try: 142 | json = r.json() 143 | next_ = json['next'] 144 | for datum in json['results']: 145 | csv_writer.writerow([datum[field] for field in fields]) 146 | print 'Page %d of %d' % (page, (json['count'] / 20) + 1) 147 | page += 1 148 | except Exception as e: 149 | print e 150 | 151 | submissions = pd.read_csv('assignment_responses.csv') 152 | submissions.update(pd.to_datetime(submissions['datetime']).dt.strftime('%D')) # Drop the time from datetime 153 | submissions_by_day = submissions.pivot_table(index = ['datetime'], columns = "crowdsource", values = "id", aggfunc='count') 154 | submissions_by_day = submissions_by_day.fillna(0).cumsum() 155 | 156 | submissions_list = [] 157 | for label in submissions_by_day: 158 | submissions_list.append([label,int(submissions_by_day[label].iloc[-1] - submissions_by_day[label].iloc[-8]),int(submissions_by_day[label].iloc[-8] - submissions_by_day[label].iloc[-15]),int(submissions_by_day[label].iloc[-1])]) 159 | return submissions_list 160 | 161 | def convertAssignmentLabel(assignmentNum): 162 | converter = { 163 | 14: "Help us build an FCC complaint Twitter bot", 164 | 35: "MuckRock Events and Talks", 165 | 24: "Join the MuckRock FOIA Slack", 166 | 25: "Monday Check In", 167 | 30: "Help explore Donald Rumsfeld's Snowflakes", 168 | 38: "Potential Stories and Records Issues", 169 | 40: "Policing the Police: Domestic Violence", 170 | 43: "Private Prison Feedback Line.", 171 | 72: "What should we know?", 172 | 73: "MuckRock Editorial Internship Application", 173 | 74: "Municipal Investments", 174 | 75: "What should we know about inmate firefighters?", 175 | 78: "Caloocan First Pass", 176 | 42: "Help us dig through the CIA's declassified archives.", 177 | 44: "Sunshine Spotlight: Massachusetts", 178 | 47: "Phone Calls from Prison", 179 | 66: "Add to the Subjects Matter: FBI Files Project", 180 | 79: "FOIA 101 Tip Line", 181 | 76: "Tell us about your FOIA Fees!", 182 | 83: "Help read the Brett Kavanaugh confirmation hearing files", 183 | 85: "College Cola Contract Crowdsource", 184 | 89: "Test" 185 | } 186 | if assignmentNum in converter: 187 | print converter[assignmentNum] 188 | return converter[assignmentNum] 189 | else: 190 | return str(int(assignmentNum)) 191 | 192 | def getStats(): 193 | url = 'https://www.muckrock.com/api_v1/' 194 | 195 | next_ = url + 'statistics' 196 | 197 | fields = ( 198 | "date", 199 | "num_crowdsource_responded_users", 200 | "total_crowdsource_responses", 201 | "project_users", 202 | "requests_processing_days", 203 | "total_active_org_members", 204 | "pro_users", 205 | "total_requests", 206 | "num_crowdsource_responded_users", 207 | "total_crowdsource_responses", 208 | "total_crowdsources", 209 | "total_pages", 210 | "total_users", 211 | ) 212 | 213 | page = 1 214 | 215 | csv_file = open('stats.csv', 'w') 216 | csv_file.seek(0) 217 | csv_writer = csv.writer(csv_file) 218 | csv_writer.writerow(fields) 219 | 220 | while next_ is not None: 221 | r = requests.get(next_, headers=headers) 222 | try: 223 | json = r.json() 224 | next_ = json['next'] 225 | for datum in json['results']: 226 | csv_writer.writerow([datum[field] for field in fields]) 227 | print 'Page %d of %d' % (page, json['count'] / 20 + 1) 228 | page += 1 229 | except Exception as e: 230 | print e 231 | 232 | def emojiTranslate(confidence): 233 | if confidence == "1": 234 | return " 😵" 235 | if confidence == "3": 236 | return " 🙀" 237 | if confidence == "5": 238 | return " 😐" 239 | if confidence == "7": 240 | return " 🙂" 241 | if confidence == "10": 242 | return " 💯" 243 | return " ?" 244 | 245 | def getRecentCampaigns(): 246 | campaign_data = [] 247 | client = MailChimp(mc_api=mailChimpKey) 248 | today = datetime.date.today() 249 | margin = datetime.timedelta(days = 7) 250 | campaigns = client.campaigns.all(get_all=True) 251 | 252 | 253 | for campaign in campaigns["campaigns"]: 254 | try: 255 | if today - margin <= datetime.datetime.strptime(campaign["send_time"][:10], "%Y-%m-%d").date() <= today: 256 | print "Matching campaign:" 257 | print "URL: " + campaign["archive_url"] 258 | campaign_data.append([campaign["send_time"][:10], str(campaign['emails_sent']), campaign["recipients"]["list_name"], str(campaign["settings"]["subject_line"]), str(int(campaign['report_summary']["open_rate"]*100)) + "%", str(int(campaign['report_summary']["click_rate"]*100)) + "%", campaign["archive_url"]]) 259 | print "Current data is /n" + campaign_data 260 | except: 261 | print "Looks like no campaign date or another issue" 262 | print campaign_data 263 | return campaign_data 264 | 265 | newsletterData = getRecentCampaigns() 266 | getStats() 267 | 268 | dataForNumbers = pd.read_csv('stats.csv').set_index("date") 269 | dataForDocumentCloud = pd.read_csv('docstats.csv').set_index("day") 270 | 271 | mondayNotes.writelines("# Numbers \n") 272 | mondayNotes.writelines("## MuckRock \n") 273 | 274 | mondayNotes.writelines("| Stat | Past Week | Week Before | Weekly Average* | vs. Goal \n") 275 | mondayNotes.writelines("|---|---|---|---|---|---|---|\n") 276 | mondayNotes.writelines("| Pro Users | " + str(int(dataForNumbers.iloc[0]["pro_users"])) + " | " + str(int(dataForNumbers.iloc[7]["pro_users"])) + " | | | |\n") 277 | mondayNotes.writelines("| Organizational Users | " + str(int(dataForNumbers.iloc[0]["total_active_org_members"])) + " | " + str(int(dataForNumbers.iloc[7]["total_active_org_members"])) + " | | | | \n") 278 | mondayNotes.writelines("| Combined Premium | " + str(int(dataForNumbers.iloc[0]["total_active_org_members"] + dataForNumbers.iloc[0]["pro_users"])) + " | " + str(int(dataForNumbers.iloc[7]["total_active_org_members"] + dataForNumbers.iloc[7]["pro_users"])) + " | | " + str(int((dataForNumbers.iloc[0]["total_active_org_members"] + dataForNumbers.iloc[0]["pro_users"]) / 229 * 100)) + "% (Goal: 229) | \n") 279 | mondayNotes.writelines("| New Assignments | " + str(int(dataForNumbers.iloc[0]["total_crowdsource_responses"] - dataForNumbers.iloc[7]["total_crowdsource_responses"])) + "| " + str(int(dataForNumbers.iloc[7]["total_crowdsource_responses"] - dataForNumbers.iloc[14]["total_crowdsource_responses"])) + " | " + str(int(dataForNumbers.iloc[0]["total_crowdsource_responses"] - dataForNumbers.iloc[83]["total_crowdsource_responses"]/12)) + " | " + str(int(dataForNumbers.iloc[0]["total_crowdsource_responses"] / 3000 * 100)) + "% (" + str(int(dataForNumbers.iloc[0]["total_crowdsource_responses"])) + " out of 3000) |\n") 280 | mondayNotes.writelines("| New Assignment Users | " + str(int(dataForNumbers.iloc[0]["num_crowdsource_responded_users"] - dataForNumbers.iloc[7]["num_crowdsource_responded_users"])) + "| " + str(int(dataForNumbers.iloc[7]["num_crowdsource_responded_users"] - dataForNumbers.iloc[14]["num_crowdsource_responded_users"])) + " | " + str(int((dataForNumbers.iloc[0]["num_crowdsource_responded_users"] - dataForNumbers.iloc[83]["num_crowdsource_responded_users"]))/12) + " | | | \n") 281 | mondayNotes.writelines("| Requests | " + str(int(dataForNumbers.iloc[0]["total_requests"] - dataForNumbers.iloc[7]["total_requests"])) + "| " + str(int(dataForNumbers.iloc[7]["total_requests"] - dataForNumbers.iloc[14]["total_requests"])) + " | " + str(int((dataForNumbers.iloc[0]["total_requests"] - dataForNumbers.iloc[83]["total_requests"])/12)) + " | ||||\n") 282 | 283 | mondayNotes.writelines("## DocumentCloud \n") 284 | 285 | mondayNotes.writelines("| Stat | Past Week | Week Before | Weekly Average* | vs. Goal \n") 286 | mondayNotes.writelines("|---|---|---|---|---|---|---|\n") 287 | print "accounts is " + str(dataForDocumentCloud.iloc[0]["accounts"]) 288 | mondayNotes.writelines("| New Accounts | " + str(int(dataForDocumentCloud.iloc[0]["accounts"]-dataForDocumentCloud.iloc[7]["accounts"])) + " | " + str(int(dataForDocumentCloud.iloc[7]["accounts"]-dataForDocumentCloud.iloc[14]["accounts"])) + " | " + str(int((dataForDocumentCloud.iloc[0]["accounts"] - dataForDocumentCloud.iloc[83]["accounts"])/12)) + " | | |\n") 289 | mondayNotes.writelines("| Active Accounts | " + str(int(dataForDocumentCloud.iloc[0]["active accounts"])) + " | " + str(int(dataForDocumentCloud.iloc[7]["active accounts"])) + " | | | |\n") 290 | mondayNotes.writelines("| New Pages | " + str(int(dataForDocumentCloud.iloc[0]["total pages"]-dataForDocumentCloud.iloc[7]["total pages"])) + " | " + str(int(dataForDocumentCloud.iloc[7]["total pages"]-dataForDocumentCloud.iloc[14]["total pages"])) + " | " + str(int((dataForDocumentCloud.iloc[0]["total pages"] - dataForDocumentCloud.iloc[83]["total pages"])/12)) + " | | |\n") 291 | mondayNotes.writelines("| New Public Pages | " + str(int(dataForDocumentCloud.iloc[0]["total public pages"]-dataForDocumentCloud.iloc[7]["total public pages"])) + " | " + str(int(dataForDocumentCloud.iloc[7]["total public pages"]-dataForDocumentCloud.iloc[14]["total public pages"])) + " | " + str(int((dataForDocumentCloud.iloc[0]["total public pages"] - dataForDocumentCloud.iloc[83]["total public pages"])/12)) + " | | |\n") 292 | mondayNotes.writelines("| New Orgs | " + str(int(dataForDocumentCloud.iloc[0]["total orgs"]-dataForDocumentCloud.iloc[7]["total orgs"])) + " | " + str(int(dataForDocumentCloud.iloc[7]["total orgs"]-dataForDocumentCloud.iloc[14]["total orgs"])) + " | " + str(int((dataForDocumentCloud.iloc[0]["total orgs"] - dataForDocumentCloud.iloc[83]["total orgs"])/12)) + " | | |\n") 293 | 294 | 295 | mondayNotes.writelines("\* Weekly average for last 12 weeks\n") 296 | 297 | 298 | mondayNotes.writelines("\n---\n\n") 299 | 300 | mondayNotes.writelines("# Editorial \n") 301 | 302 | mondayNotes.writelines("| Day | Piece | Author | \n") 303 | mondayNotes.writelines("|---|---|---|\n") 304 | mondayNotes.writelines("| Monday | Release Notes | Michael and Mitch |\n") 305 | mondayNotes.writelines("| Friday | News Round Up | TBA |\n") 306 | mondayNotes.writelines("---\n\n") 307 | 308 | mondayNotes.writelines("# Newsletters \n") 309 | mondayNotes.writelines("## Last Week \n") 310 | 311 | mondayNotes.writelines("| Day | List Size | List | Subject Line | Open % | Click % | \n") 312 | mondayNotes.writelines("|---|---|---|---|---|---|\n") 313 | for newsletter in newsletterData: 314 | mondayNotes.writelines("| " + newsletter[0] + " | " + newsletter[1] + " | " + newsletter[2] + " | [" + newsletter[3] + "](" + newsletter[6] + ") | " + newsletter[4] + " | " + newsletter[5] + " |\n") 315 | 316 | mondayNotes.writelines("## This Week \n") 317 | 318 | mondayNotes.writelines("| Day | Newsletter | Editor | \n") 319 | mondayNotes.writelines("|---|---|---|\n") 320 | mondayNotes.writelines("| Monday | Subjects Matter | JPat |\n") 321 | mondayNotes.writelines("| Tuesday | Release Notes | Michael |\n") 322 | mondayNotes.writelines("| Wednesday | Site Newsletter | |\n") 323 | mondayNotes.writelines("| Friday | Slack | Cynthia |\n") 324 | 325 | mondayNotes.writelines("# Assignments \n") 326 | 327 | 328 | mondayNotes.writelines("| Assignment | Submissions Week | Week Before | Total |\n") 329 | mondayNotes.writelines("|---|---|---|---|\n") 330 | # 331 | assignment_stats = getAssignmentStats() 332 | 333 | for assignment in assignment_stats: 334 | if assignment[1] > 0 or assignment[2] > 0: 335 | mondayNotes.writelines("|" + convertAssignmentLabel(assignment[0]) + "| " + str(assignment[1]) + "|" + str(assignment[2]) +"|" + str(assignment[3]) + "|\n") 336 | # 337 | # 338 | 339 | mondayNotes.writelines("---\n\n") 340 | mondayNotes.writelines("\n# Weekly Tasks \n") 341 | 342 | 343 | for staffer in staff_names: 344 | staffer_data = getWeeklyUpdates(staffer) 345 | mondayNotes.writelines("\n## " + staffer + "\n") 346 | mondayNotes.writelines("* " + staffer_data.goal1 + ": ") 347 | mondayNotes.writelines(emojiTranslate(str(staffer_data.goal1data[0])) + "\n") 348 | mondayNotes.writelines("* " + staffer_data.goal2 + ": ") 349 | mondayNotes.writelines(emojiTranslate(str(staffer_data.goal2data[0])) + "\n") 350 | mondayNotes.writelines("* " + staffer_data.goal3 + ": ") 351 | mondayNotes.writelines(emojiTranslate(str(staffer_data.goal3data[0])) + "\n") 352 | 353 | mondayNotes.writelines("\n### Last week's Tasks \n") 354 | for task in staffer_data.previousTasks: 355 | mondayNotes.writelines("* " + task + "\n") 356 | mondayNotes.writelines("\n### This week's Tasks \n") 357 | for task in staffer_data.tasks: 358 | mondayNotes.writelines("* " + task + "\n") 359 | 360 | 361 | mondayNotes.close() 362 | 363 | #markdownFile = open('mondaynotes.md', 'r') 364 | #html = markdown.markdown(markdownFile.read()) 365 | #mondayNotes.close() 366 | 367 | #text_file = open("report.html", "w") 368 | #text_file.write(html) 369 | #open .text_file.close() 370 | --------------------------------------------------------------------------------