tag to be 111 | an empty-element tag (it's not in 112 | HTMLBuilder.empty_element_tags). This means an empty
tag 113 | will be presented as "
", not "". 114 | 115 | The default implementation has no opinion about which tags are 116 | empty-element tags, so a tag will be presented as an 117 | empty-element tag if and only if it has no contents. 118 | "A bold statement.
" 31 | with warnings.catch_warnings(record=True) as w: 32 | soup = self.soup(markup, parse_only=strainer) 33 | self.assertEqual( 34 | soup.decode(), self.document_for(markup)) 35 | 36 | self.assertTrue( 37 | "the html5lib tree builder doesn't support parse_only" in 38 | str(w[0].message)) 39 | 40 | def test_correctly_nested_tables(self): 41 | """html5lib inserts tags where other parsers don't.""" 42 | markup = ('Here's another table:"
45 | '
| ')
48 |
49 | self.assertSoupEquals(
50 | markup,
51 | '
Here\'s another table:'
52 | '
|
| Foo |
| Bar |
| Baz |
foo
68 | 69 | ''' 70 | soup = self.soup(markup) 71 | # Verify that we can reach thetag; this means the tree is connected. 72 | self.assertEqual(b"
foo
", soup.p.encode()) 73 | 74 | def test_reparented_markup(self): 75 | markup = 'foo
\n' 76 | soup = self.soup(markup) 77 | self.assertEqual("foo
\n", soup.body.decode()) 78 | self.assertEqual(2, len(soup.find_all('p'))) 79 | 80 | 81 | def test_reparented_markup_ends_with_whitespace(self): 82 | markup = 'foo
\n\n' 83 | soup = self.soup(markup) 84 | self.assertEqual("foo
\n\n", soup.body.decode()) 85 | self.assertEqual(2, len(soup.find_all('p'))) 86 | 87 | def test_processing_instruction(self): 88 | """Processing instructions become comments.""" 89 | markup = b"""""" 90 | soup = self.soup(markup) 91 | assert str(soup).startswith("") 92 | -------------------------------------------------------------------------------- /Capstone/Pagerank/bs4/tests/test_html5lib.py.bak: -------------------------------------------------------------------------------- 1 | """Tests to ensure that the html5lib tree builder generates good trees.""" 2 | 3 | import warnings 4 | 5 | try: 6 | from bs4.builder import HTML5TreeBuilder 7 | HTML5LIB_PRESENT = True 8 | except ImportError, e: 9 | HTML5LIB_PRESENT = False 10 | from bs4.element import SoupStrainer 11 | from bs4.testing import ( 12 | HTML5TreeBuilderSmokeTest, 13 | SoupTest, 14 | skipIf, 15 | ) 16 | 17 | @skipIf( 18 | not HTML5LIB_PRESENT, 19 | "html5lib seems not to be present, not testing its tree builder.") 20 | class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest): 21 | """See ``HTML5TreeBuilderSmokeTest``.""" 22 | 23 | @property 24 | def default_builder(self): 25 | return HTML5TreeBuilder() 26 | 27 | def test_soupstrainer(self): 28 | # The html5lib tree builder does not support SoupStrainers. 29 | strainer = SoupStrainer("b") 30 | markup = "A bold statement.
" 31 | with warnings.catch_warnings(record=True) as w: 32 | soup = self.soup(markup, parse_only=strainer) 33 | self.assertEqual( 34 | soup.decode(), self.document_for(markup)) 35 | 36 | self.assertTrue( 37 | "the html5lib tree builder doesn't support parse_only" in 38 | str(w[0].message)) 39 | 40 | def test_correctly_nested_tables(self): 41 | """html5lib insertsHere's another table:"
45 | '
| ')
48 |
49 | self.assertSoupEquals(
50 | markup,
51 | '
Here\'s another table:'
52 | '
|
| Foo |
| Bar |
| Baz |
foo
68 | 69 | ''' 70 | soup = self.soup(markup) 71 | # Verify that we can reach thetag; this means the tree is connected. 72 | self.assertEqual(b"
foo
", soup.p.encode()) 73 | 74 | def test_reparented_markup(self): 75 | markup = 'foo
\n' 76 | soup = self.soup(markup) 77 | self.assertEqual(u"foo
\n", soup.body.decode()) 78 | self.assertEqual(2, len(soup.find_all('p'))) 79 | 80 | 81 | def test_reparented_markup_ends_with_whitespace(self): 82 | markup = 'foo
\n\n' 83 | soup = self.soup(markup) 84 | self.assertEqual(u"foo
\n\n", soup.body.decode()) 85 | self.assertEqual(2, len(soup.find_all('p'))) 86 | 87 | def test_processing_instruction(self): 88 | """Processing instructions become comments.""" 89 | markup = b"""""" 90 | soup = self.soup(markup) 91 | assert str(soup).startswith("") 92 | -------------------------------------------------------------------------------- /Capstone/Pagerank/bs4/tests/test_htmlparser.py: -------------------------------------------------------------------------------- 1 | """Tests to ensure that the html.parser tree builder generates good 2 | trees.""" 3 | 4 | from pdb import set_trace 5 | import pickle 6 | from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest 7 | from bs4.builder import HTMLParserTreeBuilder 8 | 9 | class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): 10 | 11 | @property 12 | def default_builder(self): 13 | return HTMLParserTreeBuilder() 14 | 15 | def test_namespaced_system_doctype(self): 16 | # html.parser can't handle namespaced doctypes, so skip this one. 17 | pass 18 | 19 | def test_namespaced_public_doctype(self): 20 | # html.parser can't handle namespaced doctypes, so skip this one. 21 | pass 22 | 23 | def test_builder_is_pickled(self): 24 | """Unlike most tree builders, HTMLParserTreeBuilder and will 25 | be restored after pickling. 26 | """ 27 | tree = self.soup("foo") 28 | dumped = pickle.dumps(tree, 2) 29 | loaded = pickle.loads(dumped) 30 | self.assertTrue(isinstance(loaded.builder, type(tree.builder))) 31 | 32 | 33 | -------------------------------------------------------------------------------- /Capstone/Pagerank/bs4/tests/test_lxml.py: -------------------------------------------------------------------------------- 1 | """Tests to ensure that the lxml tree builder generates good trees.""" 2 | 3 | import re 4 | import warnings 5 | 6 | try: 7 | import lxml.etree 8 | LXML_PRESENT = True 9 | LXML_VERSION = lxml.etree.LXML_VERSION 10 | except ImportError as e: 11 | LXML_PRESENT = False 12 | LXML_VERSION = (0,) 13 | 14 | if LXML_PRESENT: 15 | from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML 16 | 17 | from bs4 import ( 18 | BeautifulSoup, 19 | BeautifulStoneSoup, 20 | ) 21 | from bs4.element import Comment, Doctype, SoupStrainer 22 | from bs4.testing import skipIf 23 | from bs4.tests import test_htmlparser 24 | from bs4.testing import ( 25 | HTMLTreeBuilderSmokeTest, 26 | XMLTreeBuilderSmokeTest, 27 | SoupTest, 28 | skipIf, 29 | ) 30 | 31 | @skipIf( 32 | not LXML_PRESENT, 33 | "lxml seems not to be present, not testing its tree builder.") 34 | class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): 35 | """See ``HTMLTreeBuilderSmokeTest``.""" 36 | 37 | @property 38 | def default_builder(self): 39 | return LXMLTreeBuilder() 40 | 41 | def test_out_of_range_entity(self): 42 | self.assertSoupEquals( 43 | "foo�bar
", "foobar
") 44 | self.assertSoupEquals( 45 | "foo�bar
", "foobar
") 46 | self.assertSoupEquals( 47 | "foo�bar
", "foobar
") 48 | 49 | # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this 50 | # test if an old version of lxml is installed. 51 | 52 | @skipIf( 53 | not LXML_PRESENT or LXML_VERSION < (2,3,5,0), 54 | "Skipping doctype test for old version of lxml to avoid segfault.") 55 | def test_empty_doctype(self): 56 | soup = self.soup("") 57 | doctype = soup.contents[0] 58 | self.assertEqual("", doctype.strip()) 59 | 60 | def test_beautifulstonesoup_is_xml_parser(self): 61 | # Make sure that the deprecated BSS class uses an xml builder 62 | # if one is installed. 63 | with warnings.catch_warnings(record=True) as w: 64 | soup = BeautifulStoneSoup("") 65 | self.assertEqual("", str(soup.b)) 66 | self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message)) 67 | 68 | @skipIf( 69 | not LXML_PRESENT, 70 | "lxml seems not to be present, not testing its XML tree builder.") 71 | class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest): 72 | """See ``HTMLTreeBuilderSmokeTest``.""" 73 | 74 | @property 75 | def default_builder(self): 76 | return LXMLTreeBuilderForXML() 77 | -------------------------------------------------------------------------------- /Capstone/Pagerank/bs4/tests/test_lxml.py.bak: -------------------------------------------------------------------------------- 1 | """Tests to ensure that the lxml tree builder generates good trees.""" 2 | 3 | import re 4 | import warnings 5 | 6 | try: 7 | import lxml.etree 8 | LXML_PRESENT = True 9 | LXML_VERSION = lxml.etree.LXML_VERSION 10 | except ImportError, e: 11 | LXML_PRESENT = False 12 | LXML_VERSION = (0,) 13 | 14 | if LXML_PRESENT: 15 | from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML 16 | 17 | from bs4 import ( 18 | BeautifulSoup, 19 | BeautifulStoneSoup, 20 | ) 21 | from bs4.element import Comment, Doctype, SoupStrainer 22 | from bs4.testing import skipIf 23 | from bs4.tests import test_htmlparser 24 | from bs4.testing import ( 25 | HTMLTreeBuilderSmokeTest, 26 | XMLTreeBuilderSmokeTest, 27 | SoupTest, 28 | skipIf, 29 | ) 30 | 31 | @skipIf( 32 | not LXML_PRESENT, 33 | "lxml seems not to be present, not testing its tree builder.") 34 | class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): 35 | """See ``HTMLTreeBuilderSmokeTest``.""" 36 | 37 | @property 38 | def default_builder(self): 39 | return LXMLTreeBuilder() 40 | 41 | def test_out_of_range_entity(self): 42 | self.assertSoupEquals( 43 | "foo�bar
", "foobar
") 44 | self.assertSoupEquals( 45 | "foo�bar
", "foobar
") 46 | self.assertSoupEquals( 47 | "foo�bar
", "foobar
") 48 | 49 | # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this 50 | # test if an old version of lxml is installed. 51 | 52 | @skipIf( 53 | not LXML_PRESENT or LXML_VERSION < (2,3,5,0), 54 | "Skipping doctype test for old version of lxml to avoid segfault.") 55 | def test_empty_doctype(self): 56 | soup = self.soup("") 57 | doctype = soup.contents[0] 58 | self.assertEqual("", doctype.strip()) 59 | 60 | def test_beautifulstonesoup_is_xml_parser(self): 61 | # Make sure that the deprecated BSS class uses an xml builder 62 | # if one is installed. 63 | with warnings.catch_warnings(record=True) as w: 64 | soup = BeautifulStoneSoup("") 65 | self.assertEqual(u"", unicode(soup.b)) 66 | self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message)) 67 | 68 | @skipIf( 69 | not LXML_PRESENT, 70 | "lxml seems not to be present, not testing its XML tree builder.") 71 | class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest): 72 | """See ``HTMLTreeBuilderSmokeTest``.""" 73 | 74 | @property 75 | def default_builder(self): 76 | return LXMLTreeBuilderForXML() 77 | -------------------------------------------------------------------------------- /Capstone/Pagerank/force.css: -------------------------------------------------------------------------------- 1 | circle.node { 2 | stroke: #fff; 3 | stroke-width: 1.5px; 4 | } 5 | 6 | line.link { 7 | stroke: #999; 8 | stroke-opacity: .6; 9 | } 10 | -------------------------------------------------------------------------------- /Capstone/Pagerank/force.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |If you don't see a chart above, check the JavaScript console. You may 16 | need to use a different browser.
17 | 18 | 19 | -------------------------------------------------------------------------------- /Capstone/Pagerank/force.js: -------------------------------------------------------------------------------- 1 | var width = 600, 2 | height = 600; 3 | 4 | var color = d3.scale.category20(); 5 | 6 | var dist = (width + height) / 4; 7 | 8 | var force = d3.layout.force() 9 | .charge(-120) 10 | .linkDistance(dist) 11 | .size([width, height]); 12 | 13 | function getrank(rval) { 14 | return (rval/2.0) + 3; 15 | } 16 | 17 | function getcolor(rval) { 18 | return color(rval); 19 | } 20 | 21 | var svg = d3.select("#chart").append("svg") 22 | .attr("width", width) 23 | .attr("height", height); 24 | 25 | function loadData(json) { 26 | force 27 | .nodes(json.nodes) 28 | .links(json.links); 29 | 30 | var k = Math.sqrt(json.nodes.length / (width * height)); 31 | 32 | force 33 | .charge(-10 / k) 34 | .gravity(100 * k) 35 | .start(); 36 | 37 | var link = svg.selectAll("line.link") 38 | .data(json.links) 39 | .enter().append("line") 40 | .attr("class", "link") 41 | .style("stroke-width", function(d) { return Math.sqrt(d.value); }); 42 | 43 | var node = svg.selectAll("circle.node") 44 | .data(json.nodes) 45 | .enter().append("circle") 46 | .attr("class", "node") 47 | .attr("r", function(d) { return getrank(d.rank); } ) 48 | .style("fill", function(d) { return getcolor(d.rank); }) 49 | .on("dblclick",function(d) { 50 | if ( confirm('Do you want to open '+d.url) ) 51 | window.open(d.url,'_new',''); 52 | d3.event.stopPropagation(); 53 | }) 54 | .call(force.drag); 55 | 56 | node.append("title") 57 | .text(function(d) { return d.url; }); 58 | 59 | force.on("tick", function() { 60 | link.attr("x1", function(d) { return d.source.x; }) 61 | .attr("y1", function(d) { return d.source.y; }) 62 | .attr("x2", function(d) { return d.target.x; }) 63 | .attr("y2", function(d) { return d.target.y; }); 64 | 65 | node.attr("cx", function(d) { return d.x; }) 66 | .attr("cy", function(d) { return d.y; }); 67 | }); 68 | 69 | } 70 | loadData(spiderJson); 71 | -------------------------------------------------------------------------------- /Capstone/Pagerank/pagerank orginal.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Capstone/Pagerank/pagerank orginal.PNG -------------------------------------------------------------------------------- /Capstone/Pagerank/spdump.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | 3 | conn = sqlite3.connect('spider.sqlite') 4 | cur = conn.cursor() 5 | 6 | cur.execute('''SELECT COUNT(from_id) AS inbound, old_rank, new_rank, id, url 7 | FROM Pages JOIN Links ON Pages.id = Links.to_id 8 | WHERE html IS NOT NULL 9 | GROUP BY id ORDER BY inbound DESC''') 10 | 11 | count = 0 12 | for row in cur : 13 | if count < 50 : print(row) 14 | count = count + 1 15 | print(count, 'rows.') 16 | cur.close() 17 | -------------------------------------------------------------------------------- /Capstone/Pagerank/spdump.py Dr. Chuck.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Capstone/Pagerank/spdump.py Dr. Chuck.PNG -------------------------------------------------------------------------------- /Capstone/Pagerank/spdump.py WP.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Capstone/Pagerank/spdump.py WP.PNG -------------------------------------------------------------------------------- /Capstone/Pagerank/spider.js: -------------------------------------------------------------------------------- 1 | spiderJson = {"nodes":[ 2 | {"weight":126,"rank":19.0, "id":6, "url":"https://www.washingtonpost.com"}, 3 | {"weight":7,"rank":0.1367546748927559, "id":12, "url":"https://www.washingtonpost.com/news/arts-and-entertainment/wp/2017/10/26/megyn-kelly-responds-to-those-bill-oreilly-thank-you-notes-its-right-out-of-the-playbook"}, 4 | {"weight":7,"rank":0.1367546748927559, "id":14, "url":"https://www.washingtonpost.com/news/politics/wp/2017/09/25/a-trump-judicial-pick-said-transgender-children-are-proof-that-satans-plan-is-working/?utm_term=.8e0dac432ca8"}, 5 | {"weight":12,"rank":0.1367546748927559, "id":17, "url":"https://www.washingtonpost.com/news/animalia/wp/2017/10/26/this-bears-tongue-was-monstrously-enlarged-a-team-of-vets-gave-him-a-second-chance-at-life"}, 6 | {"weight":7,"rank":0.1367546748927559, "id":18, "url":"https://www.washingtonpost.com/news/speaking-of-science/wp/2017/10/25/this-bug-is-real-and-now-so-are-your-nightmares"}, 7 | {"weight":4,"rank":0.0, "id":20, "url":"https://www.washingtonpost.com/news/early-lead/wp/2017/10/26/joe-girardi-out-as-yankees-manager-and-the-nationals-just-happen-to-have-an-opening"}, 8 | {"weight":9,"rank":0.11777961277413226, "id":22, "url":"https://www.washingtonpost.com/people/mike-debonis"}, 9 | {"weight":4,"rank":0.0, "id":25, "url":"https://www.washingtonpost.com/news/post-politics/wp/2017/10/26/trump-plans-to-declare-the-opioid-crisis-a-public-health-emergency"}, 10 | {"weight":5,"rank":0.0, "id":32, "url":"https://www.washingtonpost.com/people/samantha-schmidt"}, 11 | {"weight":5,"rank":0.0, "id":38, "url":"https://www.washingtonpost.com/people/erik-wemple"}, 12 | {"weight":5,"rank":0.0, "id":41, "url":"https://www.washingtonpost.com/news/the-watch/wp/2017/10/26/mississippi-judge-resigns-after-barring-mother-from-seeing-newborn-because-of-unpaid-court-fees"}, 13 | {"weight":7,"rank":0.2567572862009351, "id":42, "url":"https://www.washingtonpost.com/people/radley-balko"}, 14 | {"weight":8,"rank":0.0, "id":43, "url":"https://www.washingtonpost.com/news/posteverything/wp/2017/10/26/george-w-bush-is-not-the-resistance-hes-part-of-what-brought-us-trump"}, 15 | {"weight":5,"rank":0.0, "id":45, "url":"https://www.washingtonpost.com/people/damian-paletta"}, 16 | {"weight":7,"rank":0.10600165188954895, "id":47, "url":"https://www.washingtonpost.com/powerpost/booker-graham-and-elias-expected-to-testify-today-in-menendez-corruption-trial/2017/10/26/34d9219c-ba4a-11e7-9e58-e6288544af98_story.html"}, 17 | {"weight":5,"rank":0.0, "id":51, "url":"https://www.washingtonpost.com/people/anna-fifield"}, 18 | {"weight":5,"rank":0.0, "id":53, "url":"https://www.washingtonpost.com/people/william-booth"}, 19 | {"weight":7,"rank":0.018975062118623574, "id":54, "url":"https://www.washingtonpost.com/local/virginia-politics/trump-again-tweets-on-virginias-governors-race-says-northam-will-be-very-weak-on-crime/2017/10/26/4c5d5ea6-ba58-11e7-be94-fabb0f1e9ffb_story.html"}, 20 | {"weight":5,"rank":0.0, "id":57, "url":"https://www.washingtonpost.com/people/perry-stein"}, 21 | {"weight":5,"rank":0.0, "id":59, "url":"https://www.washingtonpost.com/news/wonk/wp/2017/10/26/why-mcdonalds-is-beating-out-the-fresh-healthy-competition"}, 22 | {"weight":9,"rank":0.0, "id":64, "url":"https://www.washingtonpost.com/local/obituaries/notable-deaths-so-far-this-year/2017/01/17/750ed23a-dcf5-11e6-acdf-14da832ae861_gallery.html"}, 23 | {"weight":16,"rank":1.3854428993176182, "id":66, "url":"https://www.washingtonpost.com/powerpost/gops-insurgents-step-up-campaign-against-mcconnell/2017/10/25/ec3a5af4-b9a0-11e7-9e58-e6288544af98_story.html?tid=pm_pop"}, 24 | {"weight":10,"rank":0.006418872637825229, "id":74, "url":"https://www.washingtonpost.com/crossword-puzzles"}, 25 | {"weight":4,"rank":0.0, "id":80, "url":"https://www.washingtonpost.com/news/the-fix/wp/2017/10/26/trump-flubs-another-promise-declaring-the-opioid-crisis-a-national-emergency"}, 26 | {"weight":4,"rank":0.0, "id":83, "url":"https://www.washingtonpost.com/news/worldviews/wp/2017/10/26/whats-a-scary-superstition-in-your-part-of-the-world-share-it-with-us"}, 27 | {"weight":4,"rank":0.0, "id":90, "url":"https://www.washingtonpost.com/news/posteverything/wp/2017/10/26/will-republicans-regret-enabling-a-demagogue-my-iranian-parents-did"}], 28 | "links":[ 29 | {"source":0,"target":0,"value":3}, 30 | {"source":0,"target":1,"value":3}, 31 | {"source":0,"target":2,"value":3}, 32 | {"source":0,"target":3,"value":3}, 33 | {"source":0,"target":4,"value":3}, 34 | {"source":0,"target":5,"value":3}, 35 | {"source":0,"target":6,"value":3}, 36 | {"source":0,"target":7,"value":3}, 37 | {"source":0,"target":8,"value":3}, 38 | {"source":0,"target":9,"value":3}, 39 | {"source":0,"target":10,"value":3}, 40 | {"source":0,"target":11,"value":3}, 41 | {"source":0,"target":12,"value":3}, 42 | {"source":0,"target":13,"value":3}, 43 | {"source":0,"target":14,"value":3}, 44 | {"source":0,"target":15,"value":3}, 45 | {"source":0,"target":16,"value":3}, 46 | {"source":0,"target":17,"value":3}, 47 | {"source":0,"target":18,"value":3}, 48 | {"source":0,"target":19,"value":3}, 49 | {"source":0,"target":20,"value":3}, 50 | {"source":0,"target":21,"value":3}, 51 | {"source":0,"target":22,"value":3}, 52 | {"source":0,"target":23,"value":3}, 53 | {"source":0,"target":24,"value":3}, 54 | {"source":0,"target":25,"value":3}, 55 | {"source":7,"target":0,"value":3}, 56 | {"source":7,"target":7,"value":3}, 57 | {"source":9,"target":9,"value":3}, 58 | {"source":9,"target":0,"value":3}, 59 | {"source":9,"target":21,"value":3}, 60 | {"source":17,"target":0,"value":3}, 61 | {"source":17,"target":17,"value":3}, 62 | {"source":23,"target":0,"value":3}, 63 | {"source":23,"target":23,"value":3}, 64 | {"source":11,"target":11,"value":3}, 65 | {"source":11,"target":0,"value":3}, 66 | {"source":11,"target":21,"value":3}, 67 | {"source":24,"target":0,"value":3}, 68 | {"source":24,"target":24,"value":3}, 69 | {"source":19,"target":0,"value":3}, 70 | {"source":19,"target":19,"value":3}, 71 | {"source":22,"target":22,"value":3}, 72 | {"source":22,"target":0,"value":3}, 73 | {"source":8,"target":8,"value":3}, 74 | {"source":8,"target":0,"value":3}, 75 | {"source":8,"target":21,"value":3}, 76 | {"source":14,"target":0,"value":3}, 77 | {"source":14,"target":14,"value":3}, 78 | {"source":15,"target":15,"value":3}, 79 | {"source":15,"target":0,"value":3}, 80 | {"source":15,"target":21,"value":3}, 81 | {"source":4,"target":0,"value":3}, 82 | {"source":4,"target":4,"value":3}, 83 | {"source":13,"target":13,"value":3}, 84 | {"source":13,"target":0,"value":3}, 85 | {"source":13,"target":21,"value":3}, 86 | {"source":21,"target":0,"value":3}, 87 | {"source":21,"target":21,"value":3}, 88 | {"source":5,"target":0,"value":3}, 89 | {"source":5,"target":5,"value":3}, 90 | {"source":16,"target":16,"value":3}, 91 | {"source":16,"target":0,"value":3}, 92 | {"source":16,"target":21,"value":3}, 93 | {"source":18,"target":18,"value":3}, 94 | {"source":18,"target":0,"value":3}, 95 | {"source":18,"target":21,"value":3}, 96 | {"source":25,"target":0,"value":3}, 97 | {"source":25,"target":25,"value":3}, 98 | {"source":3,"target":0,"value":3}, 99 | {"source":3,"target":3,"value":3}, 100 | {"source":3,"target":21,"value":3}, 101 | {"source":12,"target":0,"value":3}, 102 | {"source":12,"target":12,"value":3}, 103 | {"source":20,"target":0,"value":3}, 104 | {"source":20,"target":20,"value":3}, 105 | {"source":1,"target":0,"value":3}, 106 | {"source":1,"target":1,"value":3}, 107 | {"source":6,"target":6,"value":3}, 108 | {"source":6,"target":0,"value":3}, 109 | {"source":6,"target":21,"value":3}, 110 | {"source":10,"target":0,"value":3}, 111 | {"source":10,"target":10,"value":3}, 112 | {"source":10,"target":11,"value":3}, 113 | {"source":2,"target":0,"value":3}, 114 | {"source":2,"target":2,"value":3}]}; -------------------------------------------------------------------------------- /Capstone/Pagerank/spider.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import urllib.error 3 | import ssl 4 | from urllib.parse import urljoin 5 | from urllib.parse import urlparse 6 | from urllib.request import urlopen 7 | from bs4 import BeautifulSoup 8 | 9 | # Ignore SSL certificate errors 10 | ctx = ssl.create_default_context() 11 | ctx.check_hostname = False 12 | ctx.verify_mode = ssl.CERT_NONE 13 | 14 | conn = sqlite3.connect('spider.sqlite') 15 | cur = conn.cursor() 16 | 17 | cur.execute('''CREATE TABLE IF NOT EXISTS Pages 18 | (id INTEGER PRIMARY KEY, url TEXT UNIQUE, html TEXT, 19 | error INTEGER, old_rank REAL, new_rank REAL)''') 20 | 21 | cur.execute('''CREATE TABLE IF NOT EXISTS Links 22 | (from_id INTEGER, to_id INTEGER)''') 23 | 24 | cur.execute('''CREATE TABLE IF NOT EXISTS Webs (url TEXT UNIQUE)''') 25 | 26 | # Check to see if we are already in progress... 27 | cur.execute('SELECT id,url FROM Pages WHERE html is NULL and error is NULL ORDER BY RANDOM() LIMIT 1') 28 | row = cur.fetchone() 29 | if row is not None: 30 | print("Restarting existing crawl. Remove spider.sqlite to start a fresh crawl.") 31 | else : 32 | starturl = input('Enter web url or enter: ') 33 | if ( len(starturl) < 1 ) : starturl = 'http://www.dr-chuck.com/' 34 | if ( starturl.endswith('/') ) : starturl = starturl[:-1] 35 | web = starturl 36 | if ( starturl.endswith('.htm') or starturl.endswith('.html') ) : 37 | pos = starturl.rfind('/') 38 | web = starturl[:pos] 39 | 40 | if ( len(web) > 1 ) : 41 | cur.execute('INSERT OR IGNORE INTO Webs (url) VALUES ( ? )', ( web, ) ) 42 | cur.execute('INSERT OR IGNORE INTO Pages (url, html, new_rank) VALUES ( ?, NULL, 1.0 )', ( starturl, ) ) 43 | conn.commit() 44 | 45 | # Get the current webs 46 | cur.execute('''SELECT url FROM Webs''') 47 | webs = list() 48 | for row in cur: 49 | webs.append(str(row[0])) 50 | 51 | print(webs) 52 | 53 | many = 0 54 | while True: 55 | if ( many < 1 ) : 56 | sval = input('How many pages:') 57 | if ( len(sval) < 1 ) : break 58 | many = int(sval) 59 | many = many - 1 60 | 61 | cur.execute('SELECT id,url FROM Pages WHERE html is NULL and error is NULL ORDER BY RANDOM() LIMIT 1') 62 | try: 63 | row = cur.fetchone() 64 | # print row 65 | fromid = row[0] 66 | url = row[1] 67 | except: 68 | print('No unretrieved HTML pages found') 69 | many = 0 70 | break 71 | 72 | print(fromid, url, end=' ') 73 | 74 | # If we are retrieving this page, there should be no links from it 75 | cur.execute('DELETE from Links WHERE from_id=?', (fromid, ) ) 76 | try: 77 | document = urlopen(url, context=ctx) 78 | 79 | html = document.read() 80 | if document.getcode() != 200 : 81 | print("Error on page: ",document.getcode()) 82 | cur.execute('UPDATE Pages SET error=? WHERE url=?', (document.getcode(), url) ) 83 | 84 | if 'text/html' != document.info().get_content_type() : 85 | print("Ignore non text/html page") 86 | cur.execute('DELETE FROM Pages WHERE url=?', ( url, ) ) 87 | cur.execute('UPDATE Pages SET error=0 WHERE url=?', (url, ) ) 88 | conn.commit() 89 | continue 90 | 91 | print('('+str(len(html))+')', end=' ') 92 | 93 | soup = BeautifulSoup(html, "html.parser") 94 | except KeyboardInterrupt: 95 | print('') 96 | print('Program interrupted by user...') 97 | break 98 | except: 99 | print("Unable to retrieve or parse page") 100 | cur.execute('UPDATE Pages SET error=-1 WHERE url=?', (url, ) ) 101 | conn.commit() 102 | continue 103 | 104 | cur.execute('INSERT OR IGNORE INTO Pages (url, html, new_rank) VALUES ( ?, NULL, 1.0 )', ( url, ) ) 105 | cur.execute('UPDATE Pages SET html=? WHERE url=?', (memoryview(html), url ) ) 106 | conn.commit() 107 | 108 | # Retrieve all of the anchor tags 109 | tags = soup('a') 110 | count = 0 111 | for tag in tags: 112 | href = tag.get('href', None) 113 | if ( href is None ) : continue 114 | # Resolve relative references like href="/contact" 115 | up = urlparse(href) 116 | if ( len(up.scheme) < 1 ) : 117 | href = urljoin(url, href) 118 | ipos = href.find('#') 119 | if ( ipos > 1 ) : href = href[:ipos] 120 | if ( href.endswith('.png') or href.endswith('.jpg') or href.endswith('.gif') ) : continue 121 | if ( href.endswith('/') ) : href = href[:-1] 122 | # print href 123 | if ( len(href) < 1 ) : continue 124 | 125 | # Check if the URL is in any of the webs 126 | found = False 127 | for web in webs: 128 | if ( href.startswith(web) ) : 129 | found = True 130 | break 131 | if not found : continue 132 | 133 | cur.execute('INSERT OR IGNORE INTO Pages (url, html, new_rank) VALUES ( ?, NULL, 1.0 )', ( href, ) ) 134 | count = count + 1 135 | conn.commit() 136 | 137 | cur.execute('SELECT id FROM Pages WHERE url=? LIMIT 1', ( href, )) 138 | try: 139 | row = cur.fetchone() 140 | toid = row[0] 141 | except: 142 | print('Could not retrieve id') 143 | continue 144 | # print fromid, toid 145 | cur.execute('INSERT OR IGNORE INTO Links (from_id, to_id) VALUES ( ?, ? )', ( fromid, toid ) ) 146 | 147 | 148 | print(count) 149 | 150 | cur.close() 151 | -------------------------------------------------------------------------------- /Capstone/Pagerank/spider.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Capstone/Pagerank/spider.sqlite -------------------------------------------------------------------------------- /Capstone/Pagerank/spjson.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | 3 | conn = sqlite3.connect('spider.sqlite') 4 | cur = conn.cursor() 5 | 6 | print("Creating JSON output on spider.js...") 7 | howmany = int(input("How many nodes? ")) 8 | 9 | cur.execute('''SELECT COUNT(from_id) AS inbound, old_rank, new_rank, id, url 10 | FROM Pages JOIN Links ON Pages.id = Links.to_id 11 | WHERE html IS NOT NULL AND ERROR IS NULL 12 | GROUP BY id ORDER BY id,inbound''') 13 | 14 | fhand = open('spider.js','w') 15 | nodes = list() 16 | maxrank = None 17 | minrank = None 18 | for row in cur : 19 | nodes.append(row) 20 | rank = row[2] 21 | if maxrank is None or maxrank < rank: maxrank = rank 22 | if minrank is None or minrank > rank : minrank = rank 23 | if len(nodes) > howmany : break 24 | 25 | if maxrank == minrank or maxrank is None or minrank is None: 26 | print("Error - please run sprank.py to compute page rank") 27 | quit() 28 | 29 | fhand.write('spiderJson = {"nodes":[\n') 30 | count = 0 31 | map = dict() 32 | ranks = dict() 33 | for row in nodes : 34 | if count > 0 : fhand.write(',\n') 35 | # print row 36 | rank = row[2] 37 | rank = 19 * ( (rank - minrank) / (maxrank - minrank) ) 38 | fhand.write('{'+'"weight":'+str(row[0])+',"rank":'+str(rank)+',') 39 | fhand.write(' "id":'+str(row[3])+', "url":"'+row[4]+'"}') 40 | map[row[3]] = count 41 | ranks[row[3]] = rank 42 | count = count + 1 43 | fhand.write('],\n') 44 | 45 | cur.execute('''SELECT DISTINCT from_id, to_id FROM Links''') 46 | fhand.write('"links":[\n') 47 | 48 | count = 0 49 | for row in cur : 50 | # print row 51 | if row[0] not in map or row[1] not in map : continue 52 | if count > 0 : fhand.write(',\n') 53 | rank = ranks[row[0]] 54 | srank = 19 * ( (rank - minrank) / (maxrank - minrank) ) 55 | fhand.write('{"source":'+str(map[row[0]])+',"target":'+str(map[row[1]])+',"value":3}') 56 | count = count + 1 57 | fhand.write(']};') 58 | fhand.close() 59 | cur.close() 60 | 61 | print("Open force.html in a browser to view the visualization") 62 | -------------------------------------------------------------------------------- /Capstone/Pagerank/sprank.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | 3 | conn = sqlite3.connect('spider.sqlite') 4 | cur = conn.cursor() 5 | 6 | # Find the ids that send out page rank - we only are interested 7 | # in pages in the SCC that have in and out links 8 | cur.execute('''SELECT DISTINCT from_id FROM Links''') 9 | from_ids = list() 10 | for row in cur: 11 | from_ids.append(row[0]) 12 | 13 | # Find the ids that receive page rank 14 | to_ids = list() 15 | links = list() 16 | cur.execute('''SELECT DISTINCT from_id, to_id FROM Links''') 17 | for row in cur: 18 | from_id = row[0] 19 | to_id = row[1] 20 | if from_id == to_id : continue 21 | if from_id not in from_ids : continue 22 | if to_id not in from_ids : continue 23 | links.append(row) 24 | if to_id not in to_ids : to_ids.append(to_id) 25 | 26 | # Get latest page ranks for strongly connected component 27 | prev_ranks = dict() 28 | for node in from_ids: 29 | cur.execute('''SELECT new_rank FROM Pages WHERE id = ?''', (node, )) 30 | row = cur.fetchone() 31 | prev_ranks[node] = row[0] 32 | 33 | sval = input('How many iterations:') 34 | many = 1 35 | if ( len(sval) > 0 ) : many = int(sval) 36 | 37 | # Sanity check 38 | if len(prev_ranks) < 1 : 39 | print("Nothing to page rank. Check data.") 40 | quit() 41 | 42 | # Lets do Page Rank in memory so it is really fast 43 | for i in range(many): 44 | # print prev_ranks.items()[:5] 45 | next_ranks = dict(); 46 | total = 0.0 47 | for (node, old_rank) in list(prev_ranks.items()): 48 | total = total + old_rank 49 | next_ranks[node] = 0.0 50 | # print total 51 | 52 | # Find the number of outbound links and sent the page rank down each 53 | for (node, old_rank) in list(prev_ranks.items()): 54 | # print node, old_rank 55 | give_ids = list() 56 | for (from_id, to_id) in links: 57 | if from_id != node : continue 58 | # print ' ',from_id,to_id 59 | 60 | if to_id not in to_ids: continue 61 | give_ids.append(to_id) 62 | if ( len(give_ids) < 1 ) : continue 63 | amount = old_rank / len(give_ids) 64 | # print node, old_rank,amount, give_ids 65 | 66 | for id in give_ids: 67 | next_ranks[id] = next_ranks[id] + amount 68 | 69 | newtot = 0 70 | for (node, next_rank) in list(next_ranks.items()): 71 | newtot = newtot + next_rank 72 | evap = (total - newtot) / len(next_ranks) 73 | 74 | # print newtot, evap 75 | for node in next_ranks: 76 | next_ranks[node] = next_ranks[node] + evap 77 | 78 | newtot = 0 79 | for (node, next_rank) in list(next_ranks.items()): 80 | newtot = newtot + next_rank 81 | 82 | # Compute the per-page average change from old rank to new rank 83 | # As indication of convergence of the algorithm 84 | totdiff = 0 85 | for (node, old_rank) in list(prev_ranks.items()): 86 | new_rank = next_ranks[node] 87 | diff = abs(old_rank-new_rank) 88 | totdiff = totdiff + diff 89 | 90 | avediff = totdiff / len(prev_ranks) 91 | print(i+1, avediff) 92 | 93 | # rotate 94 | prev_ranks = next_ranks 95 | 96 | # Put the final ranks back into the database 97 | print(list(next_ranks.items())[:5]) 98 | cur.execute('''UPDATE Pages SET old_rank=new_rank''') 99 | for (id, new_rank) in list(next_ranks.items()) : 100 | cur.execute('''UPDATE Pages SET new_rank=? WHERE id=?''', (new_rank, id)) 101 | conn.commit() 102 | cur.close() 103 | 104 | -------------------------------------------------------------------------------- /Capstone/Pagerank/spreset.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | 3 | conn = sqlite3.connect('spider.sqlite') 4 | cur = conn.cursor() 5 | 6 | cur.execute('''UPDATE Pages SET new_rank=1.0, old_rank=0.0''') 7 | conn.commit() 8 | 9 | cur.close() 10 | 11 | print("All pages set to a rank of 1.0") 12 | -------------------------------------------------------------------------------- /Programming for Everybody/Chapter 1/Hello World.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 08:59:26 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | print("Hello World!") -------------------------------------------------------------------------------- /Programming for Everybody/Chapter 2/Assignment 2.2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:00:05 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | name = input("Enter your name") 9 | print("Hello %s" % name) -------------------------------------------------------------------------------- /Programming for Everybody/Chapter 2/Assignment 2.3.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:01:38 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | hrs = input("Enter Hours:") 9 | rate = input("Enter Rate:") 10 | cost = float(hrs)*float(rate) 11 | 12 | print("Pay: %s" % cost) 13 | 14 | -------------------------------------------------------------------------------- /Programming for Everybody/Chapter 3/Assignment 3.1.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:03:50 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | try: 9 | h = input("please input your hour:") 10 | hour = float(h) 11 | r = input("please input your rate:") 12 | rate = float(r) 13 | if hour < 0: 14 | print("Please,input your positive numberic") 15 | elif rate < 0: 16 | print("Please,input your positive numberic") 17 | elif hour > 40: 18 | print("%.2f" % (40*rate+(hour-40)*1.5*rate)) 19 | else: 20 | print("%.2f" % (hour*rate)) 21 | except: 22 | print("Please,input your numberic") 23 | -------------------------------------------------------------------------------- /Programming for Everybody/Chapter 3/Assignment 3.3.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:04:59 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | score = float(input("Enter a score between 0.0 and 1.0: ")) 9 | if score<0.0 or score>1.0: 10 | print("Error: Score out of range") 11 | if score<0.6: 12 | print("F") 13 | elif score>=0.6 and score<0.7: 14 | print("D") 15 | elif score>=0.7 and score<0.8: 16 | print("C") 17 | elif score>=0.8 and score<0.9: 18 | print("B") 19 | elif score>=0.9: 20 | print("A") 21 | -------------------------------------------------------------------------------- /Programming for Everybody/Chapter 4/Assignment 4.6.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:07:25 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | def computepay(hrs,rps): 9 | final_pay = 0.0 10 | hours = float(hrs) 11 | rate_per_hour = float(rps) 12 | if hours>40: 13 | final_pay += 40 * float(rate_per_hour) 14 | hours -= 40 15 | final_pay += hours * rate_per_hour * 1.5 16 | else: 17 | final_pay += hours * rate_per_hour 18 | 19 | return final_pay 20 | 21 | 22 | hrs = input("Enter Hours: ") 23 | rate = input("Enter Rate: ") 24 | p = computepay(hrs,rate) 25 | print(p) 26 | -------------------------------------------------------------------------------- /Programming for Everybody/Chapter 5/Assignment 5.2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:09:05 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | largest = None 9 | smallest = None 10 | while True: 11 | try: 12 | num = input("Enter a number: ") 13 | if num == "done" : break 14 | n = int(num) 15 | if largest is None: 16 | largest = n 17 | if smallest is None: 18 | smallest = n 19 | if n > largest: 20 | largest = n 21 | if n < smallest: 22 | smallest = n 23 | except: 24 | print('Invalid input') 25 | 26 | print("Maximum is", largest) 27 | print("Minimum is", smallest) 28 | -------------------------------------------------------------------------------- /Python Data Structures/Atom Editor Test.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Python Data Structures/Atom Editor Test.PNG -------------------------------------------------------------------------------- /Python Data Structures/Chapter 10/Assignment 10.2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:29:38 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | name = input("Enter file:") 9 | if len(name) < 1 : name = "mbox-short.txt" 10 | handle = open(name) 11 | a = handle.read() 12 | b = a.split("\n") 13 | d = [] 14 | for i in b: 15 | if i.startswith("From "): 16 | c = i.split(":") 17 | d.append(c[0][-2:]) 18 | d.sort() 19 | counts = {} 20 | for j in d: 21 | counts[j] = d.count(j) 22 | for k, l in counts.items(): 23 | print(k, l) 24 | -------------------------------------------------------------------------------- /Python Data Structures/Chapter 6/Assignment 6.5.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:11:42 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | text = "X-DSPAM-Confidence: 0.8475"; 9 | a = text[-6:] 10 | b = float(a) 11 | print(b) 12 | text.find(":") 13 | -------------------------------------------------------------------------------- /Python Data Structures/Chapter 7/Assignment 7.1.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:12:55 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | # Use words.txt as the file name 9 | fname = input("Enter file name: ") 10 | fh = open(fname) 11 | a = fh.read() 12 | print(a.upper().rstrip()) -------------------------------------------------------------------------------- /Python Data Structures/Chapter 7/Assignment 7.2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:17:11 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | # Use the file name mbox-short.txt as the file name 9 | fname = input("Enter file name: ") 10 | fh = open(fname) 11 | a = [] 12 | for line in fh: 13 | if line.startswith("X-DSPAM-Confidence:"): 14 | a.append(float(line[-6:])) 15 | total = 0 16 | for i in a: 17 | total = total + i 18 | mean = total/(len(a)) 19 | print("Average spam confidence:", mean) 20 | -------------------------------------------------------------------------------- /Python Data Structures/Chapter 7/words.txt: -------------------------------------------------------------------------------- 1 | Writing programs or programming is a very creative 2 | and rewarding activity You can write programs for 3 | many reasons ranging from making your living to solving 4 | a difficult data analysis problem to having fun to helping 5 | someone else solve a problem This book assumes that 6 | {\em everyone} needs to know how to program and that once 7 | you know how to program, you will figure out what you want 8 | to do with your newfound skills 9 | 10 | We are surrounded in our daily lives with computers ranging 11 | from laptops to cell phones We can think of these computers 12 | as our personal assistants who can take care of many things 13 | on our behalf The hardware in our current-day computers 14 | is essentially built to continuously ask us the question 15 | What would you like me to do next 16 | 17 | Our computers are fast and have vasts amounts of memory and 18 | could be very helpful to us if we only knew the language to 19 | speak to explain to the computer what we would like it to 20 | do next If we knew this language we could tell the 21 | computer to do tasks on our behalf that were reptitive 22 | Interestingly, the kinds of things computers can do best 23 | are often the kinds of things that we humans find boring 24 | and mind-numbing 25 | 26 | -------------------------------------------------------------------------------- /Python Data Structures/Chapter 8/Assignment 8.4.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Spyder Editor 4 | 5 | This is a temporary script file. 6 | """ 7 | 8 | get = input('Please enter file name:') 9 | handle = open(get) 10 | text = list() 11 | for line in handle: 12 | line = line.rstrip() 13 | line = line.split() 14 | for i in line: 15 | if i in text: 16 | continue 17 | else: 18 | text.append(i) 19 | text.sort() 20 | print(text) 21 | -------------------------------------------------------------------------------- /Python Data Structures/Chapter 8/Assignment 8.5.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:25:10 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | file = input('Please enter file name:') 9 | handle = open(file) 10 | count = 0 11 | for line in handle: 12 | line = line.rstrip() 13 | if not line.startswith('From '): 14 | continue 15 | line = line.split() 16 | print(line[1]) 17 | count = count+1 18 | print('There were', count, 'lines in the file with From as the first word') 19 | -------------------------------------------------------------------------------- /Python Data Structures/Chapter 8/romeo.txt: -------------------------------------------------------------------------------- 1 | But soft what light through yonder window breaks 2 | It is the east and Juliet is the sun 3 | Arise fair sun and kill the envious moon 4 | Who is already sick and pale with grief 5 | 6 | -------------------------------------------------------------------------------- /Python Data Structures/Chapter 9/Assignment 9.4.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 09:27:17 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | file = input('Please enter file name: ') 9 | handle = open(file) 10 | data1 = list() 11 | data2 = dict() 12 | for line in handle: 13 | line = line.rstrip() 14 | if not line.startswith('From '): 15 | continue 16 | line = line.split() 17 | line = line[1] 18 | data1.append(line) 19 | for i in data1: 20 | data2[i] = data2.get(i,0)+1 21 | 22 | word = None 23 | max = None 24 | 25 | for aa, bb in data2.items(): 26 | if max is None or bb > max: 27 | word = aa 28 | max = bb 29 | 30 | print(word, max) -------------------------------------------------------------------------------- /Python Data Structures/Directory Test.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Python Data Structures/Directory Test.PNG -------------------------------------------------------------------------------- /Python Data Structures/Test.py: -------------------------------------------------------------------------------- 1 | print("I am writing a line on code!") 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python-for-Everybody-Coursera 2 | Coursera courses for the Python for Everybody Specialization by the University of Michigan. This specialization teaches the fundamentals on how to get started on learning to use Python. I for myself started out in a non-technical background and found a way to learn the material. 3 | -------------------------------------------------------------------------------- /Using Databases with Python/Week 2/First Database.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Using Databases with Python/Week 2/First Database.db -------------------------------------------------------------------------------- /Using Databases with Python/Week 2/First Database.db.sqbpro: -------------------------------------------------------------------------------- 1 |About this Map
45 |46 | This is a cool map from 47 | www.py4e.com. 48 |
49 | 50 | 51 | -------------------------------------------------------------------------------- /Using Databases with Python/Week 5/where.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Using Databases with Python/Week 5/where.png -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 1/Atom Editor Test.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Using Python to Access Web Data/Week 1/Atom Editor Test.PNG -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 1/Directory Test.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atse0612/Python-for-Everybody-Coursera/60eac4010931892f89e442497c5f7f3cd043831a/Using Python to Access Web Data/Week 1/Directory Test.PNG -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 1/Test.py: -------------------------------------------------------------------------------- 1 | print("I am writing a line on code!") 2 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 2/Week 2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 20:06:30 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | import re 9 | pile = open('regex_sum_41647.txt') 10 | gold = pile.read() 11 | copier = re.findall("[0-9]+", gold) 12 | dice = [int(i) for i in copier] 13 | sum = 0 14 | for k in dice: 15 | sum += k 16 | print(sum) 17 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 3/Week 3.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 20:27:29 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | import socket 9 | mysock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 10 | mysock.connect(('data.pr4e.org', 80)) 11 | cmd = 'GET http://data.pr4e.org/intro-short.txt HTTP/1.0\r\n\r\n'.encode() 12 | mysock.send(cmd) 13 | 14 | lit = list() 15 | while True: 16 | data = mysock.recv(512) 17 | lit.append(data) 18 | if (len(data) < 1): 19 | break 20 | print(data.decode()) 21 | mysock.close() 22 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 3/intro-short.txt: -------------------------------------------------------------------------------- 1 | Why should you learn to write programs? 2 | 3 | Writing programs (or programming) is a very creative 4 | and rewarding activity. You can write programs for 5 | many reasons, ranging from making your living to solving 6 | a difficult data analysis problem to having fun to helping 7 | someone else solve a problem. This book assumes that 8 | everyone needs to know how to program, and that once 9 | you know how to program you will figure out what you want 10 | to do with your newfound skills. 11 | 12 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 4/Following Links in HTML Using BeautifulSoup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 21:12:49 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | import urllib.request, urllib.parse, urllib.error 9 | from bs4 import BeautifulSoup 10 | import ssl 11 | ctx = ssl.create_default_context() 12 | ctx.check_hostname = False 13 | ctx.verify_mode = ssl.CERT_NONE 14 | 15 | url = input('Enter URL: ') 16 | num = input('Enter count: ') 17 | pos = input('Enter position: ') 18 | print('Retrieving: ', url) 19 | for times in range(int(num)): 20 | html = urllib.request.urlopen(url, context=ctx).read() 21 | soup = BeautifulSoup(html, 'html.parser') 22 | tags = soup('a') 23 | print('Retrieving: ', tags[int(pos)-1].get('href', None)) 24 | url = tags[int(pos)-1].get('href', None) 25 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 4/Scraping HTML Data with BeautifulSoup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 21:10:37 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | from urllib.request import urlopen 9 | from bs4 import BeautifulSoup 10 | import ssl 11 | 12 | ctx = ssl.create_default_context() 13 | ctx.check_hostname = False 14 | ctx.verify_mode = ssl.CERT_NONE 15 | html = urlopen(' http://py4e-data.dr-chuck.net/comments_41649.html', context=ctx).read() 16 | soup = BeautifulSoup(html, "html.parser") 17 | tags = soup('span') 18 | sum = 0 19 | coun = 0 20 | print('Enter - ') 21 | for tag in tags: 22 | coun += 1 23 | sum += int(tag.contents[0]) 24 | print('Count', coun, '\nSum', sum) 25 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 5/Extracting Data from XML.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 21:43:48 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | import urllib.request, urllib.parse, urllib.error 9 | import xml.etree.ElementTree as ET 10 | import ssl 11 | ctx = ssl.create_default_context() 12 | ctx.check_hostname = False 13 | ctx.verify_mode = ssl.CERT_NONE 14 | 15 | url = input('Enter location: ') 16 | print ('Retrieving ', url) 17 | html = urllib.request.urlopen(url, context=ctx).read() 18 | print ('Retrieved', len(html), 'characters') 19 | tree = ET.fromstring(html) 20 | print ('Count: ',len(tree.findall('.//count'))) 21 | total = 0 22 | for r in tree.findall("./comments/comment"): 23 | total += int(r.find('count').text) 24 | print ('Sum: ', total) 25 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 6/Extracting Data from JSON.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 21:45:18 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | import urllib.request, urllib.parse, urllib.error 9 | import json 10 | 11 | url = input('Enter location: ') 12 | data = urllib.request.urlopen(url).read() 13 | info = json.loads(data) 14 | info = info['comments'] 15 | print ('Retrieving', url, '\nRetrieved', len(data), 'caracters', '\nCount:', len(info)) 16 | num = 0 17 | for item in info: 18 | num += int(item['count']) 19 | print ('Sum:', num) 20 | -------------------------------------------------------------------------------- /Using Python to Access Web Data/Week 6/GEOSON API.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Oct 16 21:46:57 2017 4 | 5 | @author: atse 6 | """ 7 | 8 | import urllib.request, urllib.parse, urllib.error 9 | import json 10 | 11 | serviceurl = 'http://python-data.dr-chuck.net/geojson' 12 | address = input('Enter location: ') 13 | url = serviceurl + '?' + urllib.parse.urlencode({'sensor':'false', 'address': address}) 14 | data = urllib.request.urlopen(url).read().decode() 15 | info = json.loads(data) 16 | info = info['results'] 17 | print ('Retrieving', url, '\nRetrieved', len(data), 'caracters') 18 | for item in info: 19 | key = item['place_id'] 20 | print ('Place id:', key) 21 | --------------------------------------------------------------------------------