├── README.md ├── autoCustomRules.py ├── cosineSimilarity.py ├── doc.txt ├── docs ├── arewaSummary.txt ├── article summary.txt ├── article.txt ├── featureList.txt ├── fuzzyResult.txt ├── nltk fig speech.txt ├── obasanjo.docx.pages ├── rules.txt ├── steps & guide.txt ├── summaryObasanjo.txt └── sun_summary.txt ├── ifeanyi ├── SUN NEWS PAPER VERSION.docx ├── The Arewa Youths Consultative Forum.docx └── project material 1.docx └── summerize.py /README.md: -------------------------------------------------------------------------------- 1 | # Text summerization using fuzzy logic 2 | 3 | This is an implementation of a text summeration from a txt file using fuzzy logic 4 | 5 | ## Language & Package 6 | * python 3.6 7 | * skfuzzy from sci-kit fuzzy 8 | * nltk 9 | 10 | 11 | 12 | ### preprocessing 13 | 14 | * Lemmatization 15 | * Stopwords 16 | * sentence tokenizing 17 | 18 | ## Fuzzifier 19 | getting features from the extracted sentence 20 | 21 | ### Features Extraction 22 | 23 | * Tight weight 24 | * Sentence location 25 | * Sentence length 26 | * Thematic word 27 | * Term weight 28 | * Sentence similarity 29 | * Proper noun 30 | * Numerical data 31 | 32 | ### Membership function 33 | 34 | * Triangular membership function 35 | * Auto membership function 36 | 37 | ### Inference Engine 38 | * Rule based 39 | 40 | ## Defuzifier 41 | Converting the fuzzified values to crisp for sentence selection 42 | 43 | 44 | ## Senetence Selection 45 | * compression rate is 20%, which means.. get top 20% sentence with highest score from the list of all the sentence in the corpus. -------------------------------------------------------------------------------- /cosineSimilarity.py: -------------------------------------------------------------------------------- 1 | import math 2 | from collections import Counter 3 | 4 | 5 | def build_sentence_vector(sentence1, sentence2): 6 | iterable1 = sentence1.split() 7 | iterable2 = sentence2.split() 8 | 9 | 10 | counter1 = Counter(iterable1) 11 | counter2 = Counter(iterable2) 12 | all_items = set(counter1.keys()).union(set(counter2.keys())) 13 | vector1 = [counter1[k] for k in all_items] 14 | vector2 = [counter2[k] for k in all_items] 15 | return vector1, vector2 16 | 17 | 18 | def cosineSimilarity(v1, v2): 19 | dot_product = sum(n1 * n2 for n1, n2 in zip(v1, v2) ) 20 | magnitude1 = math.sqrt(sum(n ** 2 for n in v1)) 21 | magnitude2 = math.sqrt(sum(n ** 2 for n in v2)) 22 | return dot_product / (magnitude1 * magnitude2) 23 | 24 | 25 | fuzzy_summary_file = open("docs/summaryObasanjo.txt","r") 26 | fuzzy_content = fuzzy_summary_file.read() 27 | 28 | arewa_summary_file = open("docs/arewaSummary.txt","r") 29 | arewa_content = arewa_summary_file.read() 30 | 31 | sun_summary_file = open("docs/sun_summary.txt","r") 32 | sun_content = sun_summary_file.read() 33 | 34 | original_article_file = open("doc.txt","r") 35 | original_content = original_article_file.read() 36 | 37 | 38 | 39 | 40 | 41 | v1, v2 = build_sentence_vector(fuzzy_content, arewa_content) 42 | res1 = cosineSimilarity(v1, v2) 43 | 44 | print("Cosine similarity between fuzzy & Arewa content : ", res1) 45 | 46 | v3, v4 = build_sentence_vector(fuzzy_content, sun_content) 47 | res2 = cosineSimilarity(v3, v4) 48 | print("Cosine similarity between fuzzy & sun content : ", res2) 49 | 50 | 51 | v5, v6 = build_sentence_vector(arewa_content, sun_content) 52 | res3 = cosineSimilarity(v5, v6) 53 | print("Cosine similarity arewa content & sun content : ", res3) 54 | 55 | v7, v8 = build_sentence_vector(original_content, fuzzy_content) 56 | res4 = cosineSimilarity(v7, v8) 57 | print("Cosine similarity between original content & fuzzy result : ", res4) 58 | 59 | v9, v10 = build_sentence_vector(original_content, sun_content) 60 | res5 = cosineSimilarity(v9, v10) 61 | print("Cosine similarity between original content & sun content : ", res5) 62 | 63 | v11, v12 = build_sentence_vector(original_content, arewa_content) 64 | res6 = cosineSimilarity(v11, v12) 65 | print("Cosine similarity original content & arewa content : ", res6) 66 | -------------------------------------------------------------------------------- /doc.txt: -------------------------------------------------------------------------------- 1 | Full text: Obasanjo’s letter to Buhari 2 | Published January 24, 2018 3 | Share Tweet  Share Pin it +1 4 | 5 | 6 | Obasanjo visits Buhari 7 | THE WAY OUT: A CLARION CALL FOR COALITION FOR NIGERIA MOVEMENT
Special Press Statement
By
President Olusegun Obasanjo
———————————————————————————————-
Since we are still in the month of January, it is appropriate to wish all Nigerians Happy 2018. I am constrained to issue this special statement at this time considering the situation of the country. Some of you may be asking, “What has brought about this special occasion of Obasanjo issuing a Special Statement?” You will be right to ask such a question. But there is a Yoruba saying that ‘when lice abound in your clothes, your fingernails will never be dried of blood’. When I was in the village, to make sure that lice die, you put them between two fingernails and press hard to ensure they die and they always leave blood stains on the fingernails. To ensure you do not have blood on your fingernails, you have to ensure that lice are not harboured anywhere within your vicinity. 8 | The lice of poor performance in government – poverty, insecurity, poor economic management, nepotism, gross dereliction of duty, condonation of misdeed – if not outright encouragement of it, lack of progress and hope for the future, lack of national cohesion and poor management of internal political dynamics and widening inequality – are very much with us today. With such lice of general and specific poor performance and crying poverty with us, our fingers will not be dry of ‘blood’. 9 | Four years ago when my PDP card was torn, I made it abundantly clear that I quit partisan politics for aye but my concern and interest in Nigeria, Africa and indeed in humanity would not wane. Ever since, I have adhered strictly to that position. Since that time, I have devoted quality time to the issue of zero hunger as contained in Goal No. 2 of the Sustainable Development Goals of the UN. We have set the target that Nigeria with the participating States in the Zero Hunger Forum should reach Zero Hunger goal by 2025 – five years earlier than the UN target date. I am involved in the issue of education in some States and generally in the issue of youth empowerment and employment. I am involved in all these domestically and altruistically to give hope and future to the seemingly hopeless and those in despair. I believe strongly that God has endowed Nigeria so adequately that no Nigerian should be either in want or in despair. 10 | I believe in team work and collaborative efforts. At the international level, we have worked with other world leaders to domicile the apparatus for monitoring and encouraging socio-economic progress in Africa in our Presidential Library. The purpose of Africa Progress Group, which is the new name assumed by Africa Progress Panel (APP), is to point out where, when and what works need to be done for the progress of Africa separately and collectively by African leaders and their development partners. I have also gladly accepted the invitation of the UN Secretary-General to be a member of his eighteen-member High-Level Board of Advisers on Mediation. There are other assignments I take up in other fora for Africa and for the international community. For Africa to move forward, Nigeria must be one of the anchor countries, if not the leading anchor country. It means that Nigeria must be good at home to be good outside. No doubt, our situation in the last decade or so had shown that we are not good enough at home; hence we are invariably absent at the table that we should be abroad. 11 | All these led me to take the unusual step of going against my own political Party, PDP, in the last general election to support the opposite side. I saw that action as the best option for Nigeria. As it has been revealed in the last three years or so, that decision and the subsequent collective decision of Nigerians to vote for a change was the right decision for the nation. For me, there was nothing personal, it was all in the best interest of Nigeria and, indeed, in the best interest of Africa and humanity at large. Even the horse rider then, with whom I maintain very cordial, happy and social relationship today has come to realise his mistakes and regretted it publicly and I admire his courage and forthrightness in this regard. He has a role to play on the side line for the good of Nigeria, Africa and humanity and I will see him as a partner in playing such a role nationally and internationally, but not as a horse rider in Nigeria again. 12 | The situation that made Nigerians to vote massively to get my brother Jonathan off the horse is playing itself out again. First, I thought I knew the point where President Buhari is weak and I spoke and wrote about it even before Nigerians voted for him and I also did vote for him because at that time it was a matter of “any option but Jonathan” (aobj). But my letter to President Jonathan titled: “Before It Is Too Late” was meant for him to act before it was too late. He ignored it and it was too late for him and those who goaded him into ignoring the voice of caution. I know that praise-singers and hired attackers may be raised up against me for verbal or even physical attack but if I can withstand undeserved imprisonment and was ready to shed my blood by standing for Nigeria, I will consider no sacrifice too great to make for the good of Nigeria at any time. No human leader is expected to be personally strong or self-sufficient in all aspects of governance. 13 | I knew President Buhari before he became President and said that he is weak in the knowledge and understanding of the economy but I thought that he could make use of good Nigerians in that area that could help. Although, I know that you cannot give what you don’t have and that economy does not obey military order. You have to give it what it takes in the short-, medium- and long-term. Then, it would move. I know his weakness in understanding and playing in the foreign affairs sector and again, there are many Nigerians that could be used in that area as well. They have knowledge and experience that could be deployed for the good of Nigeria. There were serious allegations of round-tripping against some inner caucus of the Presidency which would seem to have been condoned. I wonder if such actions do not amount to corruption and financial crime, then what is it? Culture of condonation and turning blind eye will cover up rather than clean up. And going to justice must be with clean hands. 14 | I thought President Buhari would fight corruption and insurgency and he must be given some credit for his achievement so far in these two areas although it is not yet uhuru! 15 | The herdsmen/crop farmers issue is being wittingly or unwittingly allowed to turn sour and messy. It is no credit to the Federal Government that the herdsmen rampage continues with careless abandon and without finding an effective solution to it. And it is a sad symptom of insensitivity and callousness that some Governors, a day after 73 victims were being buried in a mass grave in Benue State without condolence, were jubilantly endorsing President Buhari for a second term! The timing was most unfortunate. The issue of herdsmen/crop farmers dichotomy should not be left on the political platform of blame game; the Federal Government must take the lead in bringing about solution that protects life and properties of herdsmen and crop farmers alike and for them to live amicably in the same community. 16 | 17 | But there are three other areas where President Buhari has come out more glaringly than most of us thought we knew about him. One is nepotic deployment bordering on clannishness and inability to bring discipline to bear on errant members of his nepotic court. This has grave consequences on performance of his government to the detriment of the nation. It would appear that national interest was being sacrificed on the altar of nepotic interest. What does one make of a case like that of Maina: collusion, condonation, ineptitude, incompetence, dereliction of responsibility or kinship and friendship on the part of those who should have taken visible and deterrent disciplinary action? How many similar cases are buried, ignored or covered up and not yet in the glare of the media and the public? The second is his poor understanding of the dynamics of internal politics. This has led to wittingly or unwittingly making the nation more divided and inequality has widened and become more pronounced. It also has effect on general national security. The third is passing the buck. For instance, blaming the Governor of the Central Bank for devaluation of the naira by 70% or so and blaming past governments for it, is to say the least, not accepting one’s own responsibility. Let nobody deceive us, economy feeds on politics and because our politics is depressing, our economy is even more depressing today. If things were good, President Buhari would not need to come in. He was voted to fix things that were bad and not engage in the blame game. Our Constitution is very clear, one of the cardinal responsibilities of the President is the management of the economy of which the value of the naira forms an integral part. Kinship and friendship that place responsibility for governance in the hands of the unelected can only be deleterious to good government and to the nation. 18 | President Buhari’s illness called for the sympathy, understanding, prayer and patience from every sane Nigerian. It is part of our culture. Most Nigerians prayed for him while he was away sick in London for over hundred days and he gave his Deputy sufficient leeway to carry on in his absence. We all thanked God for President Buhari for coming back reasonably hale and hearty and progressing well in his recovery. But whatever may be the state of President Buhari’s health today, he should neither over-push his luck nor over-tax the patience and tolerance of Nigerians for him, no matter what his self-serving, so-called advisers, who would claim that they love him more than God loves him and that without him, there would be no Nigeria say. President Buhari needs a dignified and honourable dismount from the horse. He needs to have time to reflect, refurbish physically and recoup and after appropriate rest, once again, join the stock of Nigerian leaders whose experience, influence, wisdom and outreach can be deployed on the side line for the good of the country. His place in history is already assured. Without impaired health and strain of age, running the affairs of Nigeria is a 25/7 affair, not 24/7. 19 | I only appeal to brother Buhari to consider a deserved rest at this point in time and at this age. I continue to wish him robust health to enjoy his retirement from active public service. President Buhari does not necessarily need to heed my advice. But whether or not he heeds it, Nigeria needs to move on and move forward. 20 | I have had occasion in the past to say that the two main political parties – APC and PDP – were wobbling. I must reiterate that nothing has happened to convince me otherwise. If anything, I am reinforced in my conviction. The recent show of PDP must give grave and great concern to lovers of Nigeria. To claim, as has been credited to the chief kingmaker of PDP, that for procuring the Supreme Court judgement for his faction of the Party, he must dictate the tune all the way and this is indeed fraught with danger. If neither APC nor PDP is a worthy horse to ride to lead Nigeria at this crucial and critical time, what then do we do? Remember Farooq Kperogi, an Associate Professor at the Kennesaw State University, Georgia, United States, calls it “a cruel Hobson’s choice; it’s like a choice between six and half a dozen, between evil and evil. Any selection or deflection would be a distinction without a difference.” We cannot just sit down lamenting and wringing our hands desperately and hopelessly. 21 | I believe the situation we are in today is akin to what and where we were in at the beginning of this democratic dispensation in 1999. The nation was tottering. People became hopeless and saw no bright future in the horizon. It was all a dark cloud politically, economically and socially. The price of oil at that time was nine dollars per barrel and we had a debt overhang of about $35 billion. Most people were confused with lack of direction in the country. One of the factors that saved the situation was a near government of national unity that was put in place to navigate us through the dark cloud. We had almost all hands on deck. We used people at home and from the diaspora and we navigated through the dark cloud of those days. At that time, most people were hopelessly groping in the dark. They saw no choice, neither in the left nor in the right, and yet we were not bereft of people at home and from the diaspora that could come together to make Nigeria truly a land flowing with milk and honey. Where we are is a matter of choice but we can choose differently to make a necessary and desirable change, once again. 22 | Wherever I go, I hear Nigerians complaining, murmuring in anguish and anger. But our anger should not be like the anger of the cripple. We can collectively save ourselves from the position we find ourselves. It will not come through self-pity, fruitless complaint or protest but through constructive and positive engagement and collective action for the good of our nation and ourselves and our children and their children. We need moral re-armament and engaging togetherness of people of like-mind and goodwill to come solidly together to lift Nigeria up. This is no time for trading blames or embarking on futile argument and neither should we accept untenable excuses for non-performance. Let us accept that the present administration has done what it can do to the limit of its ability, aptitude and understanding. Let the administration and its political party platform agree with the rest of us that what they have done and what they are capable of doing is not good enough for us. They have given as best as they have and as best as they can give. Nigeria deserves and urgently needs better than what they have given or what we know they are capable of giving. To ask them to give more will be unrealistic and will only sentence Nigeria to a prison term of four years if not destroy it beyond the possibility of an early recovery and substantial growth. Einstein made it clear to us that doing the same thing and expecting a different result is the height of folly. Already, Nigerians are committing suicide for the unbearable socio-economic situation they find themselves in. And yet Nigerians love life. We must not continue to reinforce failure and hope that all will be well. It is self-deceit and self-defeat and another aspect of folly. 23 | What has emerged from the opposition has shown no better promise from their antecedents. As the leader of that Party for eight years as President of Nigeria, I can categorically say there is nothing to write home about in their new team. We have only one choice left to take us out of Egypt to the promised land. And that is the coalition of the concerned and the willing – ready for positive and drastic change, progress and involvement. Change that will give hope and future to all our youth and dignity and full participation to all our women. Our youth should be empowered to deploy their ability to learn, innovate and work energetically at ideas and concepts in which they can make their own original inputs. Youth must be part of the action today and not relegated to leadership of tomorrow which may never come. Change that will mean enhancement of living standard and progress for all. A situation where the elected will accountably govern and every Nigerian will have equal opportunity not based on kinship and friendship but based on free citizenship. 24 | Democracy is sustained and measured not by leaders doing extra-ordinary things, (invariably, leaders fail to do ordinary things very well), but by citizens rising up to do ordinary things extra-ordinarily well. Our democracy, development and progress at this juncture require ordinary citizens of Nigeria to do the extra-ordinary things of changing the course and direction of our lackluster performance and development. If leadership fails, citizens must not fail and there lies the beauty and importance of democracy. We are challenged by the current situation; we must neither adopt spirit of cowardice nor timidity let alone impotence but must be sustained by courage, determination and commitment to say and do and to persist until we achieve upliftment for Nigeria. Nothing ventured, nothing gained and we believe that our venturing will not be in vain. God of Nigeria has endowed this country adequately and our non-performance cannot be blamed on God but on leadership. God, who has given us what we need and which is potentially there, will give us leadership enablement to actualize our potentiality. 25 | The development and modernization of our country and society must be anchored and sustained on dynamic Nigerian culture, enduring values and an enchanting Nigerian dream. We must have abiding faith in our country and its role and place within the comity of nations. Today, Nigeria needs all hands on deck. All hands of men and women of goodwill must be on deck. We need all hands to move our country forward. 26 | We need a Coalition for Nigeria, CN. Such a Movement at this juncture needs not be a political party but one to which all well-meaning Nigerians can belong. That Movement must be a coalition for democracy, good governance, social and economic well-being and progress. Coalition to salvage and redeem our country. You can count me with such a Movement. Last time, we asked, prayed and worked for change and God granted our request. This time, we must ask, pray and work for change with unity, security and progress. And God will again grant us. Of course, nothing should stop such a Movement from satisfying conditions for fielding candidates for elections. But if at any stage the Movement wishes to metamorphose into candidate-sponsoring Movement for elections, I will bow out of the Movement because I will continue to maintain my non-partisan position. Coalition for Nigeria must have its headquarters in Abuja. 27 | This Coalition for Nigeria will be a Movement that will drive Nigeria up and forward. It must have a pride of place for all Nigerians, particularly for our youth and our women. It is a coalition of hope for all Nigerians for speedy, quality and equal development, security, unity, prosperity and progress. It is a coalition to banish poverty, insecurity and despair. Our country must not be oblivious to concomitant danger around, outside and ahead. Coalition for Nigeria must be a Movement to break new ground in building a united country, a socially-cohesive and moderately prosperous society with equity, equality of opportunity, justice and a dynamic and progressive economy that is self-reliant and takes active part in global division of labour and international decision-making. 28 | The Movement must work out the path of development and the trajectory of development in speed, quality and equality in the short- medium- and long-term for Nigeria on the basis of sustainability, stability, predictability, credibility, security, cooperation and prosperity with diminishing inequality. What is called for is love, commitment and interest in our country, not in self, friends and kinship alone but particularly love, compassion and interest in the poor, underprivileged and downtrodden. It is our human duty and responsibility so to do. Failure to do this will amount to a sin against God and a crime against humanity. 29 | Some may ask, what does Obasanjo want again? Obasanjo has wanted nothing other than the best for Nigeria and Nigerians and he will continue to want nothing less. And if we have the best, we will be contented whether where we live is described as palaces or huts by others and we will always give thanks to God. 30 | I, therefore, will gladly join such a Movement when one is established as Coalition for Nigeria, CN, taking Nigeria to the height God has created it to be. From now on, the Nigeria eagle must continue to soar and fly high. CN, as a Movement, will be new, green, transparent and must remain clean and always active, selflessly so. Members must be ready to make sacrifice for the nation and pay the price of being pioneers and good Nigerians for our country to play the God-assigned role for itself, for its neighbours, for its sub-region of West Africa, for its continent and for humanity in general. For me, the strength and sustainable success of CN will derive largely from the strong commitment of a population that is constantly mobilized to the rallying platform of the fact that going forward together is our best option for building a nation that will occupy its deserved place in the global community. May God continue to lead, guide and protect us. Amen. 31 | -------------------------------------------------------------------------------- /docs/arewaSummary.txt: -------------------------------------------------------------------------------- 1 | 2 | The Arewa Youths Consultative Forum, AYCF, has praised the courage of former president Olusegun Obasanjo in his letter asking President Muhammadu Buhari not to contest the 2019 presidential election. 3 | In a lengthy and strong-worded letter dispatched to the Presidential Villa on Tuesday, the former President advised Buhari to resign and go home for various reasons. 4 | Reacting to the development, the president of Arewa Youths Consultative Forum, Alhaji Yerima Shettima, who spoke with Vanguard, pointed out that what Obasanjo said was what they had seen long ago and had been drawing the president’s attention to. 5 | Shettima said though Obasanjo was not any different from Buhari, the message he sent was very germane and must be heeded to. 6 | He accused President Buhai’s “blind supporters” of being sycophantic, adding that they failed to tell him the truth. 7 | “Obasanjo is saying what people like us saw from inception. We saw it coming earlier than now. We felt patriotism does not mean if government favours you, you cannot be objective. Being objective means even if it is your father that is doing something strong, you should summon courage to say it is right or wrong. 8 | “To me, I strongly believe that the only way you can show you love Buhari is to tell him the simple truth, which we did long before now but when we advised him how to do better than the previous governments, we were tagged Buhari’s enemy by his blind followers who felt whatever he does is right. 9 | “It is not done that way because, government is like a barrack. Anybody can be there. “This is democracy. It is about the people. Your primary responsibility has to be the welfare of the people. Anything short of that is a total failure. 10 | “You cannot torture people in the name of governance. people are dying because of hardship.if You do that, you waste all the goodwill. The blind followers will follow you. 11 | “We felt that if Buhari fails, it is the north that has failed.We feel the pains of Nigerians. According to him, even though president Obasanjo did not do better when he was in office, his message cannot be ignored. 12 | “It is not because Obasanjo is better than Buhari but the message is Important. Obasanjo is the architect of what we have today. His words are important than him. 13 | “I commend him for his courage. He should have said that before now. The time Buhari should have performed he was complaining. So many people have spoken. Hali Na’Abba has spoken.They cannot all be Buhari’s enemies”, he said. 14 | Also reacting, the National President of Arewa Youths Forum, AYF, Alhaji Gambo Gujungu, told the newspaper that the forum had earlier urged Buhari not to contest the 2019 presidency owing to the fact that he was yet to fulfill his 2015 campaign promises to Nigerians. 15 | According to him, the president’s prosecution of corrupt persons was lopsided, adding that the president cannot claim to be fighting corruption when his ‘cabals’ were allegedly enmeshed in the act. 16 | He said, “Two weeks ago, we issued a statement warning him(Buhari) not to contest the 2019 general election because he is yet to fulfill his 2015 election campaign promises. 17 | “We are also not in support of how he is fighting corruption in the country. Imagine what is going even within the so-called ‘cabals’ where issues bothering on corruption are dangling over their heads. 18 | “We are in line with what former President Obasanjo said. He (Buhari) should give way since he cannot fix the country and allow others with the capacity to do so come on board. Let him go home and rest in 2019.” 19 | -------------------------------------------------------------------------------- /docs/article summary.txt: -------------------------------------------------------------------------------- 1 | Blockchains are digital organisms. 2 | As organisms evolve through changes in their DNA, blockchain protocols evolve through changes in their code. 3 | And like biological organisms, the most adaptive blockchains will be the ones that survive and thrive. 4 | So what makes for a strong evolutionary process in a blockchain? 5 | Forking is an important mechanism that is becoming more common. -------------------------------------------------------------------------------- /docs/article.txt: -------------------------------------------------------------------------------- 1 | Funding the Evolution of Blockchains 2 | 3 | Blockchains are digital organisms. As organisms evolve through changes in their DNA, blockchain protocols evolve through changes in their code. And like biological organisms, the most adaptive blockchains will be the ones that survive and thrive. 4 | 5 | So what makes for a strong evolutionary process in a blockchain? Forking is an important mechanism that is becoming more common. Forks can speed evolution by allowing many different approaches to be tried in parallel. However, I feel there is another which is barely talked about: the economic incentives to contribute to a blockchain’s core protocols. These incentives are lacking in almost every major blockchain today and are an opportunity to supercharge their development. 6 | 7 | The lack of incentives to work on core protocols is reflected in the large number of people working on Etheruem tokens vs. the small number working on Ethereum itself. Launching a new token has made many millionaires in the last 8 months, whether on paper or liquid. Meanwhile, if you contribute to the core Ethereum codebase at best you 1) own a bunch of ETH personally and the price of them goes up a bit or 2) need to join the Ethereum Foundation and get paid some amount that wouldn’t match the economics of a successful token launch. As a result, Ethereum is starting to suffer from a tragedy of the commons problem: while lots of people own ETH and would benefit from Ethereum improving, the economic reward for any single individual improving it is low. 8 | 9 | So it’s not surprising these massively valuable blockchains don’t have many people working on them. Despite being worth over $30bn and $65bn, there are only 15 meaningful contributors to Ethereum and Bitcoin respectively, and the rate of contribution isn’t going up much with their rise in popularity. 10 | 11 | This is a shame. Improvements to these protocols would create massive amounts of value. For example, let’s say someone or a group of people implemented an Ethereum scaling solution like sharding or Plasma. Each of these improvements would likely increase the value of Ethereum by over 10%, creating roughly $3 billion in value at current Ether prices. 12 | 13 | So how do we incent people to work on the common good of the core protocols? 14 | 15 | Private funding was one of the first methods used when MIT funded Bitcoin core developers Gavin Andresen, Wladimir van der Laan, and Cory Fields in 2015. Some projects like Blockstack have funds of private individuals and VCs to invest in projects built on top of the protocol, but this is different because it is not focused on improving the core protocol directly. 16 | 17 | Private funding can be a good first step because it is fast, decisive, and lightweight. However, the economics probably don’t work at some point. If a protocol upgrade like sharding in Ethereum could create $3bn in value, private funding is already unable to scale to be able to offer anything close to the value that could be created. 18 | 19 | Public crowdfunding hasn’t been tried much in the blockchain space or nothing has worked enough to come to mind. A theoretical example of this would be the Ethereum Foundation announcing an open crowdfund for a feature bounty. 20 | 21 | Public crowdfunding still suffers from a tragedy of the commons problem. Everyone will want the benefit of the crowdfunded efforts but is incentivized to sit on the sidelines and hope others chip in. 22 | 23 | Inflation funding is where things get interesting. It allows economic rewards that are otherwise unthinkable. Remember, if Ether holders believed an upgrade (ex: sharding) would make the price go up by >10%, they’d be happy to pay close to 10% of their tokens for it. That means Ethereum could crowdfund a $3bn feature bounty by inflating the number of ETH by 10% and pay the newly created tokens to the creator(s) of the upgrade. This is somewhat analogous to taxes: everyone in the community chips in to fund common infrastructure (ex: roads) which no one would build alone. 24 | 25 | Funding protocol developments through inflation would also allow anyone in the world to contribute more easily. Not everyone wants to be employed by a foundation or reveal their identity: some of the largest advances in cryptocurrency like Bitcoin itself and Mimblewimble were anonymously airdropped. 26 | 27 | Finally, the funds in token sales set aside for future protocol development will eventually run out, and this feels like a good solution to that eventuality. 28 | 29 | An example of how this would work: a developer submits a pull request to Ethereum with a working implementation of sharding. The community discusses and tests it. There is an on-chain vote of Ether holders to 1) determine if the upgrade should be merged and 2) if so, what the size of the bounty should be. The pull request increases the number of ETH outstanding and sends the new tokens to an address the developer includes. This would require a hard fork the first time, but may not in the future. The size of the bounty ends up being fair because Ether holders want to incentivize developers to keep submitting improvements like this in the future. 30 | 31 | I find Tezos interesting because it proposes a version of this concept. Here’s an excerpt from their position paper: 32 | “Funding Innovation”, from the Tezos position paper 33 | 34 | Open feature bounties are a powerful way of accelerating change. The self-driving car revolution was kicked off by The DARPA Grand Challenge to make an autonomous car traverse 132mi of a desert. It offered a $2m feature bounty and drew over 40 teams from different universities and companies. Imagine how much competition there would be for multi-hundred million or billion dollar feature bounties. And the potential size of the bounties will grow as the value of the network grows. 35 | A $2m feature bounty created this. 36 | 37 | A protocol which provides strong incentives for people to improve it is likely to evolve faster than one that does not. So blockchains which fund innovation through token inflation would seem to have a superior evolutionary algorithm. And over the long run, rate of change is often more important than starting point. 38 | 39 | There are some counterarguments. Contributors value things beyond money: the intellectual pursuit, the status and camaraderie it brings within their community, and the mission of a project. But I’d argue proper economic incentives can only add to this list and are also necessary for some people to be able to focus 100% of their time on a project. Also, as The Mythical Man Month suggests, more people working on software doesn’t necessarily mean it will go faster or turn out better. However, adding more people to software projects usually fails when its components can’t be divided up, and there are a sufficient number of different scaling paths at this point where each can be pursued relatively independently. Finally, approving upgrades and bounties requires greater community coordination. Yet, looking at different attempts to upgrade protocols, it seems on-chain voting on upgrades is something that would reduce complexity, not add it. 40 | 41 | So, perhaps the highest leverage thing protocol designers can do is think about how to engineer the evolutionary characteristics of their blockchains — specifically, the economic incentives for anyone to come along and improve them. The best engineered organisms can outpace others, even if they start smaller or later. 42 | 43 | This can be the biggest step function change in the rate of innovation in the blockchain space if implemented well. By harnessing their decentralized nature they can evolve faster than a centralized organization ever could. 44 | 45 | Thanks to Dan Romero, Arthur Breitman, Brian Armstrong, Joseph Poon, and Albert Wenger for conversations which influenced this post. 46 | Thanks to Dan Romero. 47 | 48 | -------------------------------------------------------------------------------- /docs/featureList.txt: -------------------------------------------------------------------------------- 1 | 0.0 0.125 1.0 0.0 0.0 0.0 0.8611746569214657 0.3545959161729105 2 | 0.3333333333333333 0.375 0.5 0.0 0.012987012987012988 0.08333333333333333 0.8323008018694606 0.5489867724320188 3 | 0.0 0.34375 0.3333333333333333 0.0 0.012987012987012988 0.0 0.890104286558 0.6135468212260897 4 | 0.3333333333333333 0.21875 0.25 0.0 0.0 0.0 0.1415628266270753 0.5024202476584305 5 | 0.0 0.1875 0.2 0.0 0.0 0.0 0.7272362455386101 0.4589930224114076 6 | 0.0 0.3125 0.16666666666666666 0.0 0.0 0.0 0.5761322576946664 0.616523324887544 7 | 0.3333333333333333 0.5 0.14285714285714285 0.0 0.012987012987012988 0.0 0.9154110369971196 0.6756524629165008 8 | 0.3333333333333333 0.375 0.125 0.0 0.0 0.0 0.5649647513178889 0.6796604363201145 9 | 0.0 0.59375 0.1111111111111111 0.0 0.03896103896103896 0.10526315789473684 0.6822791855104424 0.7843865092373005 10 | 0.0 0.4375 0.1 0.07142857142857142 0.012987012987012988 0.0 0.7479022234324645 0.6805100025666658 11 | 0.0 1.0 0.09090909090909091 0.0625 0.03896103896103896 0.125 0.7081219702351473 1.0 12 | 0.0 0.78125 0.08333333333333333 0.0 0.025974025974025976 0.12 0.8910724553299835 0.8690407121071245 13 | 0.0 0.34375 0.07692307692307693 0.0 0.025974025974025976 0.0 0.7127296299515179 0.577075025308159 14 | 0.0 0.6875 0.07142857142857142 0.13636363636363635 0.012987012987012988 0.09090909090909091 0.811432795307292 0.8204829219365453 15 | 0.0 0.09375 0.06666666666666667 0.0 0.0 0.0 0.9879529374702298 0.2899441380556921 16 | 0.0 0.25 0.0625 0.0 0.03896103896103896 0.0 0.8314833260072018 0.534361151871094 17 | 0.0 0.53125 0.058823529411764705 0.0 0.025974025974025976 0.11764705882352941 0.804095312114299 0.7630769866139792 18 | 0.0 0.625 0.05555555555555555 0.1 0.025974025974025976 0.1 0.7983341007987746 0.7525767127647286 19 | 0.0 0.28125 0.05263157894736842 0.0 0.025974025974025976 0.0 0.3053491992890376 0.5766625960086641 20 | 0.0 0.71875 0.05 0.043478260869565216 0.0 0.34782608695652173 0.7697644398788518 0.9169499886331589 21 | 0.3333333333333333 0.65625 0.047619047619047616 0.0 0.025974025974025976 0.09523809523809523 0.7327133178326426 0.8327785812751994 22 | 0.0 0.34375 0.045454545454545456 0.0 0.0 0.0 0.9903693525230266 0.5800565833589522 23 | 0.0 0.25 0.043478260869565216 0.0 0.012987012987012988 0.0 1.0 0.45992012646477654 24 | 0.0 0.78125 0.041666666666666664 0.04 0.05194805194805195 0.04 0.7626352561527837 0.8887733566310686 25 | 0.3333333333333333 0.40625 0.04 0.0 0.012987012987012988 0.07692307692307693 0.5893086085458972 0.6817425766186693 26 | 0.0 0.375 0.038461538461538464 0.0 0.025974025974025976 0.08333333333333333 0.7661518976168258 0.6410627125121581 27 | 0.0 0.25 0.037037037037037035 0.0 0.0 0.125 0.6537802657021312 0.5440594165016016 28 | 0.0 0.375 0.03571428571428571 0.0 0.0 0.0 0.5118574943296345 0.6829086704048761 29 | 0.0 0.1875 0.034482758620689655 0.0 0.0 0.0 0.736071477438524 0.4598660897758887 30 | 0.0 0.21875 0.03333333333333333 0.0 0.0 0.0 0.6722154954685728 0.503195248326442 31 | 0.0 0.84375 0.03225806451612903 0.07407407407407407 0.03896103896103896 0.07407407407407407 0.8476087471146286 0.7982763869009191 32 | 0.0 0.71875 0.03125 0.08695652173913043 0.03896103896103896 0.08695652173913043 0.6844625732183064 0.865903082090013 33 | 0.3333333333333333 0.65625 0.030303030303030304 0.0 0.012987012987012988 0.0 0.6083365306847961 0.8174238500367589 34 | 0.0 0.375 0.029411764705882353 0.0 0.025974025974025976 0.0 0.6875324821725967 0.674029687136489 35 | 0.0 0.53125 0.02857142857142857 0.0 0.012987012987012988 0.11764705882352941 0.5011082806935796 0.7951424822057243 36 | 0.3333333333333333 0.59375 0.027777777777777776 0.0 0.03896103896103896 0.0 0.9321834465241161 0.8168308305152023 37 | 0.0 0.4375 0.02702702702702703 0.0 0.012987012987012988 0.07142857142857142 0.7234305568569404 0.7057306667745735 38 | 0.0 0.15625 0.02631578947368421 0.0 0.0 0.0 0.839198741087835 0.40793779548202747 39 | 0.0 0.5 0.02564102564102564 0.125 0.025974025974025976 0.0625 0.7852984070849143 0.6783977571505407 40 | 0.0 0.4375 0.025 0.0 0.012987012987012988 0.07142857142857142 0.6083833879952852 0.7379334051156292 41 | 0.0 0.34375 0.024390243902439025 0.0 0.012987012987012988 0.0 0.909154459017294 0.6118713699717108 42 | 0.0 0.5 0.023809523809523808 0.0 0.025974025974025976 0.0625 0.6374212421216552 0.7912047958390758 43 | 0.0 0.25 0.023255813953488372 0.0 0.0 0.0 0.6216611913255589 0.504934747374529 44 | 0.0 0.6875 0.022727272727272728 0.0 0.012987012987012988 0.13636363636363635 0.6392831795412104 0.7233834940433695 45 | 0.0 0.5 0.022222222222222223 0.0625 0.0 0.1875 0.4554636200608015 0.7484069921443981 46 | 0.0 0.40625 0.021739130434782608 0.15384615384615385 0.012987012987012988 0.0 0.6416247804903877 0.6771543374295705 47 | 0.0 0.34375 0.02127659574468085 0.0 0.025974025974025976 0.09090909090909091 0.711536876000593 0.673386311448404 48 | 0.0 0.28125 0.020833333333333332 0.0 0.025974025974025976 0.0 0.7067452128272609 0.5770643053249649 49 | 0.0 0.21875 0.02040816326530612 0.14285714285714285 0.012987012987012988 0.0 0.8453347208767609 0.40845391277312937 50 | 0.0 0.375 0.02 0.0 0.025974025974025976 0.0 0.7366886734739007 0.6450094376881299 51 | 0.3333333333333333 0.375 0.0196078431372549 0.0 0.025974025974025976 0.0 0.703732418808445 0.6755829674282839 52 | 0.0 0.34375 0.019230769230769232 0.0 0.0 0.0 0.8605603024926277 0.6173156181522647 53 | 0.0 0.09375 0.018867924528301886 0.0 0.0 0.0 0.9574147274359679 0.2914793220104816 54 | 0.0 0.5625 0.018518518518518517 0.0 0.012987012987012988 0.0 0.872972504205749 0.7683391604126114 55 | 0.0 0.59375 0.01818181818181818 0.05263157894736842 0.012987012987012988 0.0 0.545700356516721 0.7925279755383614 56 | 0.0 0.59375 0.017857142857142856 0.0 0.025974025974025976 0.15789473684210525 0.895521675280627 0.7928983684283535 57 | 0.0 0.71875 0.017543859649122806 0.0 0.025974025974025976 0.0 0.8272347195448553 0.8934221789844291 58 | 0.0 0.3125 0.017241379310344827 0.0 0.025974025974025976 0.0 0.9400604879516921 0.5763312114213079 59 | 0.0 0.59375 0.01694915254237288 0.0 0.03896103896103896 0.0 0.9831413991278474 0.7820397550173361 60 | 0.0 0.71875 0.016666666666666666 0.0 0.012987012987012988 0.0 0.8582627821319444 0.8909413197187339 61 | 0.0 0.375 0.01639344262295082 0.0 0.0 0.0 0.8215921149464008 0.6485789347761326 62 | 0.3333333333333333 0.375 0.016129032258064516 0.0 0.0 0.0 0.6186077083860965 0.6809070927738271 63 | 0.0 0.34375 0.015873015873015872 0.0 0.0 0.0 0.5542570090950191 0.6507112444722413 64 | 0.0 0.59375 0.015625 0.0 0.0 0.5263157894736842 0.9208418440238677 0.7714919974293298 65 | 0.0 0.125 0.015384615384615385 0.0 0.0 0.5 0.8470177716283704 0.3573854995919821 -------------------------------------------------------------------------------- /docs/fuzzyResult.txt: -------------------------------------------------------------------------------- 1 | 0.26499699592840775 2 | 0.3928774145768274 3 | 0.2638986145774105 4 | 0.35504251385279706 5 | 0.2628506037607776 6 | 0.26326739460995346 7 | 0.3268428419511071 8 | 0.32105901687358585 9 | 0.26531741323574387 10 | 0.2996641704300476 11 | 0.29596045518133623 12 | 0.2622551388381526 13 | 0.26419612414143767 14 | 0.30118099235098905 15 | 0.26567800417357945 16 | 0.2611203771639909 17 | 0.26625308671391185 18 | 0.29310904452046954 19 | 0.2622379432724261 20 | 0.2883776342182731 21 | 0.2897091882372559 22 | 0.26419612414143767 23 | 0.2611203771639909 24 | 0.28647848353153543 25 | 0.28599497334204804 26 | 0.2650089993839994 27 | 0.2611203771639909 28 | 0.2650089993839994 29 | 0.26325167962722545 30 | 0.26222072519206746 31 | 0.2820695189451866 32 | 0.28160208856258523 33 | 0.2810141538657 34 | 0.2650089993839994 35 | 0.266540038077293 36 | 0.27966817840526553 37 | 0.2662098132564147 38 | 0.26418213231658394 39 | 0.2788476480353084 40 | 0.2662098132564147 41 | 0.26419612414143767 42 | 0.26659437536689057 43 | 0.2611203771639909 44 | 0.26323593896927244 45 | 0.27771637224994444 46 | 0.2765507480462723 47 | 0.26419612414143767 48 | 0.2622379432724261 49 | 0.2752303027530299 50 | 0.2650089993839994 51 | 0.2752683600188494 52 | 0.26419612414143767 53 | 0.26567800417357945 54 | 0.2661957957957958 55 | 0.27540448095288855 56 | 0.2656682717805527 57 | 0.2622034846440644 58 | 0.26326739460995346 59 | 0.2656682717805527 60 | 0.2622034846440644 61 | 0.2650089993839994 62 | 0.2733498461726971 63 | 0.26419612414143767 64 | 0.2656682717805527 65 | 0.26499699592840775 -------------------------------------------------------------------------------- /docs/nltk fig speech.txt: -------------------------------------------------------------------------------- 1 | ['NNS', 'JJ', 'NN', '.'] 2 | ['IN', 'NN', 'VBP', 'NN', 'NNP', ',', 'NN', 'NN', 'VBP', 'NN', 'NN', '.'] 3 | ['CC', 'IN', 'JJ', 'NN', ',', 'JJ', 'NNS', 'CD', 'JJ', 'NN', '.'] 4 | ['RB', 'VB', 'JJ', 'JJ', 'NN', 'NN', '.'] 5 | ['VBG', 'JJ', 'NN', 'VBG', 'JJ', '.'] 6 | ['NNS', 'VBD', 'NN', 'VBG', 'JJ', 'JJ', 'NN', 'VBD', 'JJ', '.'] 7 | ['RB', ',', 'PRP', 'VBP', 'DT', 'RB', 'VBN', ':', 'JJ', 'NN', 'NN', 'NN', 'POS', 'NN', 'NN', '.'] 8 | ['DT', 'NN', 'VBG', 'RB', 'DT', 'JJ', 'NN', 'NN', 'NN', 'NN', 'NN', '.'] 9 | ['DT', 'NN', 'JJ', 'NN', 'NN', 'NN', 'VBD', 'JJ', 'NN', 'NNS', 'VBG', 'NNP', 'JJ', 'FW', 'JJ', 'NN', 'VBG', 'NNP', '.'] 10 | ['VBG', 'JJ', 'NNS', 'VBD', 'JJ', 'NN', 'JJ', 'CD', 'NN', ',', 'IN', 'NN', 'NN', '.'] 11 | ['RB', ',', 'NN', 'NN', 'NNP', 'NN', 'JJS', 'CD', ')', 'NN', 'NNP', 'RB', 'NN', 'VBP', 'RB', 'CD', ')', 'NN', 'NN', 'NNP', 'NNP', 'VB', 'VBN', 'NN', 'MD', 'RB', 'VB', 'NNS', 'JJ', 'JJ', 'NN', '.'] 12 | ['IN', 'NN', ',', 'NNP', 'VBG', 'NN', 'NN', 'JJ', 'NN', ':', 'NN', 'NNS', 'NNP', 'MD', 'VB', 'NNP', 'NN', ',', 'JJ', 'NN', 'JJ', 'JJ', 'NN', 'JJ', '.'] 13 | ['RB', 'POS', 'JJ', 'RB', 'JJ', 'NNS', 'RB', 'JJ', 'NNS', 'VBG', '.'] 14 | ['IN', 'JJ', '$', 'CD', '$', 'CD', ',', 'CD', 'JJ', 'NN', 'NNP', 'NNP', 'RB', ',', 'NN', 'NN', 'RB', 'VBG', 'JJ', 'NN', 'NN', '.'] 15 | ['DT', 'NN', '.'] 16 | ['NNS', 'NN', 'MD', 'VB', 'JJ', 'NN', 'NN', '.'] 17 | ['IN', 'NN', ',', 'VB', 'POS', 'VB', 'NN', 'NN', 'NNS', 'VBD', 'NNP', 'VBG', 'NN', 'IN', 'VBG', 'NNP', '.'] 18 | ['DT', 'NN', 'MD', 'RB', 'VB', 'NN', 'NNP', 'CD', 'NN', ',', 'VBG', 'RB', '$', 'CD', 'CD', 'NN', 'JJ', 'NNP', 'NN', '.'] 19 | ['RB', 'JJ', 'NNS', 'VBP', 'JJ', 'JJ', 'NN', 'NN', '.'] 20 | ['JJ', 'NN', 'CD', 'JJ', 'NN', 'VBN', 'NNP', 'VBD', 'NNP', 'NN', 'NN', 'NNP', 'NNP', ',', 'NNP', 'FW', 'NN', 'NNP', ',', 'NNP', 'NNP', 'CD', '.'] 21 | ['DT', 'NN', 'IN', 'NNP', 'NN', 'JJ', 'JJ', 'NNP', 'NN', 'NN', 'VBN', 'JJ', 'NN', ',', 'JJ', 'VBD', 'VBG', 'NN', 'NN', 'RB', '.'] 22 | ['JJ', 'NN', 'JJ', 'JJ', 'NN', 'RB', ',', 'JJ', ',', 'JJ', '.'] 23 | ['RB', ',', 'NNS', 'RB', 'RB', 'VB', 'NN', '.'] 24 | ['IN', 'JJ', 'NN', 'IN', 'VBG', 'NNP', 'MD', 'VB', '$', 'CD', 'NN', ',', 'JJ', 'NN', 'RB', 'JJ', 'NN', 'JJ', 'NN', 'NN', 'JJ', 'NN', 'MD', 'VB', '.'] 25 | ['NNP', 'VBG', 'RB', 'VBD', 'JJ', 'NN', 'NN', 'NN', 'VBD', 'JJ', 'NN', 'NN', '.'] 26 | ['DT', 'JJ', 'NN', 'MD', 'VB', 'NNP', 'VBG', 'JJ', 'NN', 'NN', 'NN', '.'] 27 | ['NNP', 'NN', 'RB', 'VBZ', 'JJ', 'JJ', 'NN', '.'] 28 | ['NN', 'VBP', 'NN', 'VBD', 'NN', 'VBN', 'JJ', 'NN', 'NN', 'NNS', 'NN', '.'] 29 | ['NN', 'VBG', 'NN', 'NN', 'NN', '.'] 30 | ['PRP', 'VBZ', 'JJ', 'NN', 'RB', 'JJ', '.'] 31 | ['NNP', ',', 'NNP', 'NN', 'VBN', 'JJ', '(', 'NN', ':', 'NN', ')', 'MD', 'VB', 'NN', 'VB', 'JJ', 'CD', 'NN', ',', 'MD', 'JJ', 'VB', 'RB', 'CD', 'NN', 'NN', '.'] 32 | ['DT', 'VBP', 'NNP', 'MD', 'VB', '$', 'CD', 'NN', 'NN', 'VBG', 'NN', 'NNP', 'CD', 'NN', 'NN', 'RB', 'VBN', 'JJ', 'NN', '(', ')', 'NN', '.'] 33 | ['DT', 'RB', 'JJ', 'NN', ':', 'NN', 'NN', 'NN', 'NN', 'JJ', 'NN', '(', 'NN', ':', 'NN', ')', 'CD', 'MD', 'VB', 'RB', '.'] 34 | ['VBG', 'NN', 'NN', 'NN', 'MD', 'RB', 'VB', 'NN', 'NN', 'NN', 'RB', '.'] 35 | ['RB', 'NN', 'VBP', 'VBN', 'NN', 'NN', 'NN', ':', 'JJS', 'NN', 'NN', 'IN', 'NNP', 'NNP', 'RB', 'VBD', '.'] 36 | ['RB', ',', 'NN', 'JJ', 'NN', 'VBN', 'RB', 'JJ', 'NN', 'NN', 'RB', 'VBP', ',', 'VBP', 'IN', 'JJ', 'NN', 'NN', '.'] 37 | ['DT', 'NN', 'MD', 'VB', ':', 'NN', 'VBZ', 'JJ', 'NN', 'NNP', 'VBG', 'NN', 'NN', '.'] 38 | ['DT', 'NN', 'NN', 'NN', '.'] 39 | ['EX', 'JJ', 'NN', 'NNP', 'NN', 'CD', ')', 'NN', 'VBD', 'VBD', 'CD', ')', ',', 'NN', 'NN', '.'] 40 | ['DT', 'NN', 'NN', 'NN', 'NN', 'NNP', 'JJ', 'VBZ', 'JJ', 'JJ', 'NN', 'NN', 'VBZ', '.'] 41 | ['DT', 'MD', 'VB', 'JJ', 'NN', 'JJ', 'NN', ',', 'MD', 'VB', '.'] 42 | ['DT', 'NN', 'JJ', 'NN', 'NN', 'NNP', 'NN', 'VBP', 'VB', 'JJ', 'VB', 'VBG', 'NN', 'IN', 'NN', '.'] 43 | ['PRP', 'VBP', 'JJ', 'JJ', 'NNS', 'NN', 'NN', '.'] 44 | ['RB', 'VBZ', 'JJ', 'NN', 'NN', ':', '``', 'NNP', 'NNP', '``', ',', 'NNP', 'NN', 'NN', 'JJ', 'NN', 'NN', 'JJ', 'NN', 'VBG', 'NN', '.'] 45 | ['DT', 'JJ', 'NN', 'NN', 'VBD', 'DT', 'NNP', 'NNP', 'NNP', 'VBP', 'JJ', 'NN', 'NN', 'CD', 'NN', '.'] 46 | ['PRP', 'VBD', '$', 'CD', 'NN', 'NN', 'VBD', 'CD', 'NN', 'JJ', 'NN', 'NN', '.'] 47 | ['NNP', 'JJ', 'NN', 'MD', 'CD', 'CD', 'CD', 'NN', 'NN', 'NN', '.'] 48 | ['CC', 'JJ', 'NN', 'NN', 'NN', 'NN', 'NN', 'NNS', '.'] 49 | ['DT', '$', 'CD', 'NN', 'NN', 'VBD', '.'] 50 | ['DT', 'NN', 'VBZ', 'JJ', 'NN', 'NNS', 'VBP', 'JJ', 'VBP', 'RBR', 'CD', '.'] 51 | ['RB', 'VBZ', 'NN', 'NN', 'JJ', 'NN', 'MD', 'VB', 'JJ', 'JJ', 'NN', '.'] 52 | ['CC', 'JJ', 'NN', ',', 'NN', 'NN', 'RB', 'JJ', 'VBG', 'NN', '.'] 53 | ['EX', 'NN', '.'] 54 | ['NNS', 'NN', 'NN', 'IN', 'NN', ':', 'JJ', 'NN', ',', 'NN', 'NN', 'NNS', 'IN', 'NN', ',', 'NN', 'NN', '.'] 55 | ['CC', 'PRP', 'MD', 'VB', 'JJ', 'JJ', 'NN', 'NN', 'NN', 'RB', 'JJ', 'NNS', 'JJ', 'RB', 'CD', 'NN', 'NN', 'NN', '.'] 56 | ['RB', ',', 'DT', 'NNP', 'NNP', 'NNP', 'VBZ', ',', 'NNS', 'VBG', 'NN', 'RB', 'RB', 'VB', 'VBP', 'RBR', 'VBP', 'RBR', '.'] 57 | ['RB', ',', 'VBG', 'NNS', 'NN', 'NN', 'RB', 'VBZ', 'NN', 'MD', 'RB', 'VB', ',', 'JJ', 'NN', 'JJ', 'VBG', 'NN', 'NN', 'VBD', 'RB', 'RB', '.'] 58 | ['RB', ',', 'VBG', 'JJ', 'NN', 'VBZ', 'JJR', 'NN', 'NN', '.'] 59 | ['RB', ',', 'VBG', 'JJ', 'NN', 'NN', 'NN', ',', 'VBZ', 'JJ', 'NN', 'JJ', 'NN', 'MD', 'VB', 'NN', ',', 'NN', '.'] 60 | ['RB', ',', 'RB', 'JJS', 'JJ', 'NN', 'NN', 'NN', 'VBP', 'NN', 'JJ', 'JJ', 'NNS', 'VBP', 'RB', ',', 'JJ', 'NN', 'NN', 'VBN', 'IN', 'NN', '.'] 61 | ['DT', 'JJS', 'JJ', 'NN', 'NN', 'NNS', ',', 'RB', 'VBP', 'JJR', 'RB', '.'] 62 | ['DT', 'JJS', 'NN', 'NN', 'NN', 'NN', 'NN', 'NN', 'NN', 'VBN', 'RB', '.'] 63 | ['IN', 'VBG', 'JJ', 'NN', 'VBP', 'JJR', 'VBN', 'NN', 'RB', 'MD', '.'] 64 | ['NNS', 'NNP', 'NNP', ',', 'NNP', 'NNP', ',', 'NNP', 'NNP', ',', 'NNP', 'NNP', ',', 'NNP', 'NNP', 'NN', 'VBD', 'NN', '.'] 65 | ['NNS', 'NNP', 'NNP', '.'] -------------------------------------------------------------------------------- /docs/obasanjo.docx.pages: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princesegzy01/text-summerization-using-fuzzy-logic/adc8f7e72a75a6cd0a7b7ee4e1bce6c731d2cf1a/docs/obasanjo.docx.pages -------------------------------------------------------------------------------- /docs/steps & guide.txt: -------------------------------------------------------------------------------- 1 | Feature list 2 | 1. title word 3 | 2. sentence length 4 | 3. sentence_location 5 | 4. numerical_data 6 | 5. thematic key 7 | 6. proper_noun 8 | 7. sentence_similarity 9 | 8. term_weight 10 | 11 | themeatic word : 10 12 | compression rate : 20 13 | 14 | universe = crisp value betwen 0 and 1 step of 0001 15 | 16 | Antecedent is the rule 17 | result = autoMembershipfunction(3) // divide it into 3 i.e Good average poor 18 | 19 | 20 | result = triangular membership function (universe, a,b,b) 21 | 22 | 23 | Rule Consequent Triangular membership fuction 24 | low = fuzzy.trimf(result, [0.000,0.30,0.500]) 25 | medium = fuzzy.trimf(result, [0.300,0.500,0.700]) 26 | high = fuzzy.trimf(result, [0.500,1.0,1.0]) 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/summaryObasanjo.txt: -------------------------------------------------------------------------------- 1 | Full text: Obasanjo’s letter to Buhari 2 | 3 | Published January 24, 2018Share Tweet Share Pin it +1 4 | Obasanjo visits Buhari 5 | THE WAY OUT: A CLARION CALL FOR COALITION FOR NIGERIA MOVEMENT
Special Press Statement By President Olusegun Obasanjo
 6 | Since we are still in the month of January, it is appropriate to wish all Nigerians Happy 2018.Four years ago when my PDP card was torn, I made it abundantly clear that I quit partisan politics for aye but my concern and interest in Nigeria, Africa and indeed in humanity would not wane. 7 | 8 | 2 of the Sustainable Development Goals of the UN.We have set the target that Nigeria with the participating States in the Zero Hunger Forum should reach Zero Hunger goal by 2025 – five years earlier than the UN target date. 9 | 10 | All these led me to take the unusual step of going against my own political Party, PDP, in the last general election to support the opposite side. 11 | 12 | Even the horse rider then, with whom I maintain very cordial, happy and social relationship today has come to realise his mistakes and regretted it publicly and I admire his courage and forthrightness in this regard.He has a role to play on the side line for the good of Nigeria, Africa and humanity and I will see him as a partner in playing such a role nationally and internationally, but not as a horse rider in Nigeria again. 13 | But my letter to President Jonathan titled: "Before It Is Too Late" was meant for him to act before it was too late. 14 | 15 | I knew President Buhari before he became President and said that he is weak in the knowledge and understanding of the economy but I thought that he could make use of good Nigerians in that area that could help. 16 | Then, it would move. 17 | I thought President Buhari would fight corruption and insurgency and he must be given some credit for his achievement so far in these two areas although it is not yet uhuru! And it is a sad symptom of insensitivity and callousness that some Governors, a day after 73 victims were being buried in a mass grave in Benue State without condolence, were jubilantly endorsing President Buhari for a second term! 18 | The timing was most unfortunate. 19 | What does one make of a case like that of Maina: collusion, condonation, ineptitude, incompetence, dereliction of responsibility or kinship and friendship on the part of those who should have taken visible and deterrent disciplinary action? 20 | 21 | For instance, blaming the Governor of the Central Bank for devaluation of the naira by 70% or so and blaming past governments for it, is to say the least, not accepting one's own responsibility. 22 | It is part of our culture. 23 | 24 | Without impaired health and strain of age, running the affairs of Nigeria is a 25/7 affair, not 24/7. 25 | To claim, as has been credited to the chief kingmaker of PDP, that for procuring the Supreme Court judgement for his faction of the Party, he must dictate the tune all the way and this is indeed fraught with danger. 26 | 27 | I believe the situation we are in today is akin to what and where we were in at the beginning of this democratic dispensation in 1999. 28 | 29 | The nation was tottering. 30 | 31 | The price of oil at that time was nine dollars per barrel and we had a debt overhang of about $35 billion. 32 | They saw no choice, neither in the left nor in the right, and yet we were not bereft of people at home and from the diaspora that could come together to make Nigeria truly a land flowing with milk and honey. 33 | 34 | To ask them to give more will be unrealistic and will only sentence Nigeria to a prison term of four years if not destroy it beyond the possibility of an early recovery and substantial growth. 35 | 36 | Change that will give hope and future to all our youth and dignity and full participation to all our women. 37 | Democracy is sustained and measured not by leaders doing extra-ordinary things, (invariably, leaders fail to do ordinary things very well), but by citizens rising up to do ordinary things extra-ordinarily well. 38 | Our democracy, development and progress at this juncture require ordinary citizens of Nigeria to do the extra-ordinary things of changing the course and direction of our lackluster performance and development. 39 | 40 | We are challenged by the current situation; we must neither adopt spirit of cowardice nor timidity let alone impotence but must be sustained by courage, determination and commitment to say and do and to persist until we achieve upliftment for Nigeria. 41 | 42 | The development and modernization of our country and society must be anchored and sustained on dynamic Nigerian culture, enduring values and an enchanting Nigerian dream. 43 | 44 | You can count me with such a Movement. 45 | 46 | But if at any stage the Movement wishes to metamorphose into candidate-sponsoring Movement for elections, I will bow out of the Movement because I will continue to maintain my non-partisan position. 47 | What is called for is love, commitment and interest in our country, not in self, friends and kinship alone but particularly love, compassion and interest in the poor, underprivileged and downtrodden. 48 | 49 | I, therefore, will gladly join such a Movement when one is established as Coalition for Nigeria, CN, taking Nigeria to the height God has created it to be. 50 | 51 | Members must be ready to make sacrifice for the nation and pay the price of being pioneers and good Nigerians for our country to play the God-assigned role for itself, for its neighbours, for its sub-region of West Africa, for its continent and for humanity in general. 52 | Amen. -------------------------------------------------------------------------------- /docs/sun_summary.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | SUN NEWS PAPER VERSION 4 | 5 | AT a time like this, the counsel is appropriate: Concentrate on the message, and leave the messenger safely alone. So, I will resist the temptation to discuss Obasanjo and concentrate on the value of his message.
Obasanjo, as he admitted in the letter, had advertised Buhari’s so-called weaknesses on the economy and foreign relations to no end. And, yes, Buhari may be weak in those areas, but there are certainly men of quality and vision helping out in those areas. Buhari’s Economic Team, though understated and often maligned is proving to be one of the most effective yet in the nation’s history.
The failing of past governments has been clear. From a rural agricultural base at independence, the country transformed to an import-dependent economy upon the discovery of oil at the turn of the decade. Instead of growing our agricultural base and using the new wealth from oil to transform to a globally-competitive human capital and industrialized giant that we could be, we abandoned even the rudimentary agriculture that had served us very well until then and engaged in the export of crude oil for the personal aggrandizement of the new men in power and their collaborators, home and abroad.
This was the challenge that faced the country for several decades until the inception of this democracy in 1999, and which Obasanjo as president couldn’t do much to reverse unfortunately. So, when Obasanjo talks of ‘round-tripping’ in the inner circles of the present government, one wonders what exactly he means. In fact, my enquiry did not yield much apart from the mild rebuke: “You know Obasanjo knows everything!” But on reflection, the retired General must have had the humungous sums in local and foreign currencies found in the NIA vaults and in places where they should not be in mind. What he accuses the Buhari government of however has indeed been the business of every past government, and that it is much in the public domain these days means that the present one is serious about eradicating it or at least reducing it to its barest minimum.
If anyone, more so Obasanjo who knows the inner workings of government, wants to be fair, what is on display, whether with the former SGF Babachir Lawal, NIA’s Oke and now Maina’s cases, is deep rooted systemic rot which needs painstaking and proper interrogation to get to the root. Especially in the Maina case, he is certainly not alone, but in fact represents a racket that has been at the bottom of the many failed civil service reforms. A purported sack cannot be enough, no matter how quickly executed. Obasanjo, I recall, was part of the Mohammed experiment, full of fury and decisiveness, which many later adjudged to be a failure, and in fact, having the opposite effect of promoting the malaise that it sought to cure in the first place.
Buhari lived through it all. And that is why when he made his recent statement about taking his time to come to decisions and having to live with his conscience thereafter, I thought that instead of the insults he got, what he deserved was praise. This is indeed our country, and we have seen it all. The good, and the mostly bad times!
Which makes one to ask: do we really know the nuances of the system of government we have chosen and the change we voted for? Sadly, even Obasanjo betrayed his ignorance of both in his recent communication. Like God’s mills, the mills of democracy grind slowly, but unlike God’s mills, they hardly grind exceedingly well. This is the bitter truth we must tell ourselves. I remember many commentators judging Obasanjo’s first term (1999-2003) as having achieved very little. In fact, the most that could be pointed to was the reforms in the military and the liberalization of the communication sector. Of course, a lot more was cooking on the economic front, like the debt-forgiveness negotiations which benefit was reaped in his second term.
Change, by its very nature, is like birth pangs of a woman. It comes with excruciating pains and dislocations. It can be slow and gradual too. As an individual, I am perhaps one of the worst affected. In my middle years and with no gainful employment, apart from a part-time job, I admonish myself and millions of others in my shoes that we may be hungry now, but we must never fall for the trap of anger. Anger is a negative emotion which does not yield anything good and lasting. What the present situation of the country requires is a deep introspection and to understand that every great nation went through these painful birth pangs before setting on the path of irrevocable development. The often cited Lee Kwan Yew’s Singapore endured his benevolent dictatorship for 15 years before the breakthrough came; Gorbachev’s USSR swallowed the bile of Glasnost and Perestroika and the dismantling of the anachronistic USSR to break forth in the 15 wonders of freedom and progress that it has become today.
The journey that the immediate past government and the PDP as the party then in power was taking us on, could only have arrived at one destination: Golgotha! Obasanjo cannot completely absolve himself of blame from this outcome. As soon as he took the reins of office, he re-created the PDP that brought him to power in his own image. The organs of the party which are supposed to facilitate a robust generation and processing of government policy were discountenanced and completely discarded. The few, like the present agriculture minister, Audu Ogbeh who had the presence of mind to caution, were humiliated and thrown out. Under who did Chris Uba’s Anambra, Zaki-Biam and Odi massacres occur? Under whom did EFCC become a vindictive machine for achieving narrow and little political ends, despite the best efforts of the pioneer leadership of the outfit?
All this go to show that the problems we have in our polity, go beyond strong men. We need institutional and structural reforms. I am glad that stakeholders including the ruling APC which committee just submitted its report continue’s to interrogate the issue. Some of the lapses observed so far under the present Buhari administration should have proved conclusively to all doubters, if there are still any, that our country is in urgent need of real reforms.
And unlike, the maddening crowd out there, that is not something to take lightly or without deep contemplation. That is why again, instead of crucifying President Buhari for calling for more reflection on the subject in his New Year message to the country, we ought to take a step back and reflect. He may not be necessarily averse to restructuring like many career reformists would have us believe. His might rather just be a caution that we cannot run away from the reality that men run, and processes would be built on, whatever structure we eventually come up with.
Owaiye writes from Lagos. 6 | -------------------------------------------------------------------------------- /ifeanyi/SUN NEWS PAPER VERSION.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princesegzy01/text-summerization-using-fuzzy-logic/adc8f7e72a75a6cd0a7b7ee4e1bce6c731d2cf1a/ifeanyi/SUN NEWS PAPER VERSION.docx -------------------------------------------------------------------------------- /ifeanyi/The Arewa Youths Consultative Forum.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princesegzy01/text-summerization-using-fuzzy-logic/adc8f7e72a75a6cd0a7b7ee4e1bce6c731d2cf1a/ifeanyi/The Arewa Youths Consultative Forum.docx -------------------------------------------------------------------------------- /ifeanyi/project material 1.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princesegzy01/text-summerization-using-fuzzy-logic/adc8f7e72a75a6cd0a7b7ee4e1bce6c731d2cf1a/ifeanyi/project material 1.docx -------------------------------------------------------------------------------- /summerize.py: -------------------------------------------------------------------------------- 1 | from nltk.tokenize import sent_tokenize, word_tokenize 2 | import nltk 3 | import re 4 | import string 5 | import math 6 | from collections import Counter 7 | from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer,TfidfTransformer 8 | from pprint import pprint 9 | import numpy as np 10 | import skfuzzy as fuzzy 11 | import operator 12 | 13 | import numpy as np 14 | import skfuzzy.control as ctrl 15 | # import skfuzzy as fuzz 16 | import sys 17 | import autoCustomRules as acr 18 | 19 | 20 | stopwords_list = nltk.corpus.stopwords.words('english') 21 | 22 | def build_sentence_vector(sentence1, sentence2): 23 | iterable1 = sentence1.split() 24 | iterable2 = sentence2.split() 25 | 26 | 27 | counter1 = Counter(iterable1) 28 | counter2 = Counter(iterable2) 29 | all_items = set(counter1.keys()).union(set(counter2.keys())) 30 | vector1 = [counter1[k] for k in all_items] 31 | vector2 = [counter2[k] for k in all_items] 32 | return vector1, vector2 33 | 34 | 35 | def cosineSimilarity(v1, v2): 36 | dot_product = sum(n1 * n2 for n1, n2 in zip(v1, v2) ) 37 | magnitude1 = math.sqrt(sum(n ** 2 for n in v1)) 38 | magnitude2 = math.sqrt(sum(n ** 2 for n in v2)) 39 | return dot_product / (magnitude1 * magnitude2) 40 | 41 | def title_word_feature(title, document): 42 | 43 | title_word_feature_values = [] 44 | 45 | title_token = word_tokenize(title) 46 | 47 | for sentence in document : 48 | document_token = word_tokenize(sentence) 49 | 50 | res = (set(document_token).intersection(title_token)) 51 | total = len(res)/ len(title_token) 52 | 53 | title_word_feature_values.append(total) 54 | 55 | 56 | return title_word_feature_values 57 | 58 | 59 | def sentence_length_feature(sentences): 60 | 61 | sentence_legth_feature_result = [] 62 | # get longest length of the sentence 63 | 64 | longest_sentence = 0 65 | 66 | for sentence in sentences : 67 | 68 | sentence_tokenize = word_tokenize(sentence) 69 | 70 | if len(sentence_tokenize) > longest_sentence : 71 | longest_sentence = len(sentence_tokenize) 72 | 73 | # normalize each length to a list 74 | for sentence_ in sentences : 75 | sentence_tokenize = word_tokenize(sentence_) 76 | 77 | sentence_legth_feature_result.append(len(sentence_tokenize) / longest_sentence ) 78 | 79 | return sentence_legth_feature_result 80 | 81 | 82 | def get_sentence_location_feature(sentences): 83 | 84 | sentence_location_result = [] 85 | 86 | for index, sentence in enumerate(sentences) : 87 | 88 | sentence_pos = index + 1 89 | sentence_location_result.append(1 / sentence_pos) 90 | 91 | return sentence_location_result 92 | 93 | 94 | def get_sentence_numerical_data(sentences): 95 | 96 | numerical_feature = [] 97 | 98 | for sentence in sentences : 99 | 100 | sentence_tokenize = word_tokenize(sentence) 101 | 102 | result = [float(s) for s in re.findall(r'-?\d+\.?\d*', sentence)] 103 | result = len(result) 104 | sentence_length = len(sentence_tokenize) 105 | 106 | 107 | numerical_feature.append(result/sentence_length) 108 | 109 | return numerical_feature 110 | 111 | 112 | def proper_noun_feature(sentences): 113 | 114 | proper_noun_result = [] 115 | 116 | 117 | # loop through senctence in each sentences 118 | for sentence in sentences: 119 | 120 | sentence_tokenize = word_tokenize(sentence) 121 | 122 | pos_title = nltk.pos_tag(sentence_tokenize) 123 | 124 | nos_nnp_in_sentence = 0 125 | 126 | # custom 127 | #testPOS = [] 128 | 129 | for k in pos_title: 130 | 131 | 132 | # custom print 133 | # testPOS.append(k[1]) 134 | 135 | 136 | if k[1] == "NNP": 137 | nos_nnp_in_sentence = nos_nnp_in_sentence + 1 138 | 139 | # CUSTOM 140 | # print(testPOS) 141 | 142 | 143 | result = nos_nnp_in_sentence / len(sentence_tokenize) 144 | 145 | proper_noun_result.append(result) 146 | 147 | return proper_noun_result 148 | 149 | 150 | def thematic_keyword(title, sentences): 151 | 152 | document_thematic_result = [] 153 | 154 | keyword_dict = dict() 155 | 156 | # loop through each token in the title 157 | for token in title: 158 | 159 | # convert token to lower case 160 | token = token.lower() 161 | 162 | # make sure token dosent exist in pucntuation or stop word list 163 | if (token not in list(string.punctuation) and (token not in stopwords_list)): 164 | 165 | if token not in keyword_dict : 166 | keyword_dict[token] = 1 167 | elif token in keyword_dict : 168 | keyword_dict[token] = keyword_dict[token] + 1 169 | 170 | 171 | # loop through senctence in each sentences 172 | for sentence in sentences: 173 | 174 | sentence_tokenize = word_tokenize(sentence) 175 | 176 | # loop through token in each sentence 177 | for token in sentence_tokenize: 178 | token = token.lower() 179 | 180 | # make sure token dosent exist in pucntuation or stop word list 181 | if (token not in list(string.punctuation) and (token not in stopwords_list)): 182 | 183 | if token not in keyword_dict : 184 | keyword_dict[token] = 1 185 | elif token in keyword_dict : 186 | keyword_dict[token] = keyword_dict[token] + 1 187 | 188 | from operator import itemgetter 189 | result = sorted(keyword_dict.items(), key=itemgetter(1)) 190 | 191 | #thematic words 192 | thematic_word = [] 193 | 194 | # max_keyword_value = res_tuple[1] 195 | 196 | res_tuple = result[-10:] 197 | 198 | #print(res_tuple) 199 | 200 | # get all the nth thematic words from tuple 201 | for tpl in res_tuple: 202 | token_name = tpl[0] 203 | thematic_word.append(token_name) 204 | 205 | 206 | # get totla number of thematic words in sentense 207 | total_sentense_thematic = 0 208 | 209 | for sentence in sentences : 210 | sentense_tokenize = word_tokenize(sentence) 211 | result = set(sentense_tokenize).intersection(thematic_word) 212 | 213 | total_sentense_thematic = total_sentense_thematic + len(result) 214 | 215 | 216 | # get thematic result for each sentence 217 | for sentence in sentences : 218 | sentense_tokenize = word_tokenize(sentence) 219 | result = set(sentense_tokenize).intersection(thematic_word) 220 | 221 | result = (len(result) / total_sentense_thematic) 222 | 223 | 224 | document_thematic_result.append(result) 225 | return document_thematic_result 226 | 227 | 228 | def sentence_to_sentence_similarity(sentences): 229 | 230 | 231 | all_similarity = [] 232 | sentence_similarity = [] 233 | 234 | 235 | # loop through senctence in each sentences 236 | for index, sentence in enumerate(sentences): 237 | 238 | sum_similartity = 0 239 | 240 | for i, sentence_ in enumerate(sentences): 241 | 242 | if index == i : 243 | continue 244 | 245 | vector1, vector2 = build_sentence_vector(sentence, sentence_) 246 | similarity = cosineSimilarity(vector1, vector2) 247 | 248 | sum_similartity = sum_similartity + similarity 249 | 250 | all_similarity.append(sum_similartity) 251 | 252 | 253 | maxSimilarity = max(all_similarity) 254 | 255 | for similairy in all_similarity : 256 | sentence_similarity.append(similairy / maxSimilarity) 257 | 258 | return sentence_similarity 259 | 260 | 261 | def termWeight(sentences): 262 | 263 | all_tf_isf_score = [] 264 | sentences_weight = [] 265 | 266 | vectorizer = TfidfVectorizer() 267 | tfidf_matrix = vectorizer.fit_transform(sentences) 268 | 269 | sentence_length = len(sentences) 270 | 271 | for i in range(sentence_length): 272 | sentence_weight = tfidf_matrix[i].data 273 | sum_sentence_weight = sum(sentence_weight) 274 | all_tf_isf_score.append(sum_sentence_weight) 275 | 276 | max_sentence_weight = max(all_tf_isf_score) 277 | 278 | 279 | for sent_weight in all_tf_isf_score: 280 | sentences_weight.append(sent_weight/max_sentence_weight) 281 | 282 | return sentences_weight 283 | 284 | 285 | def feature_importantility(count, sentence_feature_list): 286 | 287 | # print(title_word) 288 | 289 | # print(sentence_fuzzy_result , " -- ", sentence_length) 290 | 291 | count = count + 1 292 | sent_res = [] 293 | 294 | for sent_feature in sentence_feature_list: 295 | sentence_fuzzy_result = single_feature_category(sent_feature) 296 | maxKey = max(sentence_fuzzy_result.items(), key=operator.itemgetter(1))[0] 297 | sent_res.append(maxKey) 298 | 299 | return {"S " + str(count): sent_res} 300 | 301 | 302 | def single_feature_category(features): 303 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features) 304 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features) 305 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features) 306 | return dict(low=cat_low, medium=cat_medium, high=cat_high) 307 | 308 | 309 | def all_feature_category(features_object): 310 | 311 | result = [] 312 | 313 | # Add title to array of object 314 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['title_word']) 315 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['title_word']) 316 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['title_word']) 317 | result.append({"title_word" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 318 | 319 | # Add sentence_length to array of object 320 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['sentence_length']) 321 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['sentence_length']) 322 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['sentence_length']) 323 | result.append({"sentence_length" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 324 | 325 | # Add sentence_location to array of object 326 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['sentence_location']) 327 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['sentence_location']) 328 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['sentence_location']) 329 | result.append({"sentence_location" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 330 | 331 | # Add numerical_data to array of object 332 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['numerical_data']) 333 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['numerical_data']) 334 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['numerical_data']) 335 | result.append({"numerical_data" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 336 | 337 | # Add thematic_keyword to array of object 338 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['thematic_keyword']) 339 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['thematic_keyword']) 340 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['thematic_keyword']) 341 | result.append({"thematic_keyword" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 342 | 343 | # Add proper_noun to array of object 344 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['proper_noun']) 345 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['proper_noun']) 346 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['proper_noun']) 347 | result.append({"proper_noun" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 348 | 349 | # Add sentence_similarity to array of object 350 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['sentence_similarity']) 351 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['sentence_similarity']) 352 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['sentence_similarity']) 353 | result.append({"sentence_similarity" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 354 | 355 | # Add title to array of object 356 | cat_low = fuzzy.interp_membership(features_universe, universe_low, features_object['term_weight']) 357 | cat_medium = fuzzy.interp_membership(features_universe, universe_medium, features_object['term_weight']) 358 | cat_high = fuzzy.interp_membership(features_universe, universe_high, features_object['term_weight']) 359 | result.append({"term_weight" : dict(low=cat_low, medium=cat_medium, high=cat_high)}) 360 | 361 | return result 362 | 363 | 364 | def rules_definition(sentence_fuzzy_objects): 365 | 366 | for sentence_fuzzy in sentence_fuzzy_objects : 367 | # print(sentence_fuzzy[0]['title_word']) 368 | 369 | title_word = sentence_fuzzy[0]['title_word'] 370 | sentence_length = sentence_fuzzy[1]['sentence_length'] 371 | sentence_location = sentence_fuzzy[2]['sentence_location'] 372 | numerical_data = sentence_fuzzy[3]['numerical_data'] 373 | thematic_keyword = sentence_fuzzy[4]['thematic_keyword'] 374 | proper_noun = sentence_fuzzy[5]['proper_noun'] 375 | sentence_similarity = sentence_fuzzy[6]['sentence_similarity'] 376 | term_weight = sentence_fuzzy[7]['term_weight'] 377 | 378 | 379 | 380 | 381 | # 1. If (title is High) and (Length is High) and (Term is High) and (Position is High) and (Similarity is not High) and (Noun is High) and (Thematic is High) and (Numerical is High) then (Sentence is Important) (1) 382 | # 2. If (title is High) and (Length is High) and (Term is High) and (Position is High) and (Similarity is High) and (Noun is not High) and (Thematic is High) and (Numerical is HIgh) then (Sentence is Important) (1) 383 | # 3. If (title is High) and (Length is High) and (Term is High) and (Position is High) and (Similarity is High) and (Noun is High) and (Thematic is not High) and (Numerical is HIgh) then (Sentence is Important) (1) 384 | # 4. If (title is High) and (Length is not High) and (Term is not High) and (Position is not High) and (Similarity is not High) and (Noun is not High) and (Thematic is not High) and (Numerical is not HIgh) then (Sentence is Unimportant) (1) 385 | # 5. If (title is not High) and (Length is High) and (Term is not High) and (Position is not High) and (Similarity is not High) and (Noun is not High) and (Thematic is not High) and (Numerical is not HIgh) then (Sentence is Unimportant) (1) 386 | # 6. If (title is not High) and (Length is not High) and (Term is High) and (Position is not High) and (Similarity is not High) and (Noun is not High) and (Thematic is not High) and (Numerical is not HIgh) then (Sentence is Unimportant) (1) 387 | # 7. If (title is High) and (Length is not High) and (Term is High) and (Position is High) and (Similarity is not High) and (Noun is not High) and (Thematic is not High) and (Numerical is not HIgh) then (Sentence is Average) (1) 388 | # 9. If (title is High) and (Length is High) and (Term is not High) and (Position is High) and (similarity is not High) and (Noun is not High) and (Thematic is not High) and (Numerical is not High) then (Sentence is Average) (1) 389 | 390 | rule1 = np.fmax(title_word['high'], sentence_length['high'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 391 | rule2 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['low'], thematic_keyword['high'], proper_noun['low'], sentence_similarity['low'], term_weight['high']) 392 | rule3 = np.fmax(title_word['low'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['low'], proper_noun['high'], sentence_similarity['low'], term_weight['low']) 393 | # rule4 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 394 | # rule5 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 395 | # rule6 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 396 | # rule7 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 397 | # rule8 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 398 | # rule9 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 399 | # rule10 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 400 | # rule11 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 401 | # rule12 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 402 | # rule13 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 403 | # rule14 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 404 | # rule15 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 405 | # rule16 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 406 | # rule17 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 407 | # rule18 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 408 | # rule19 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 409 | # rule20 = np.fmax(title_word['high'], sentence_length['low'], sentence_location['high'], numerical_data['high'], thematic_keyword['high'], proper_noun['high'], sentence_similarity['high'], term_weight['high']) 410 | 411 | # # Determine the weight and aggregate 412 | # rule1 = np.fmax(temp_in['hot'], hum_in['low']) 413 | # rule2 = temp_in['warm'] 414 | # rule3 = np.fmax(temp_in['warm'], hum_in['high']) 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | imp1 = np.fmin(rule1, important_sentence) 424 | imp2 = np.fmin(rule2, average_sentence) 425 | imp3 = np.fmin(rule3, umimportant_sentence) 426 | 427 | aggregate_membership = np.fmax(imp1, imp2, imp3) 428 | 429 | # # Defuzzify 430 | result_sentence = fuzzy.defuzz(output_universe, aggregate_membership, 'centroid') 431 | 432 | print(result_sentence) 433 | 434 | 435 | 436 | 437 | def fuzzy_rules(sentence_feature_object): 438 | 439 | # print(sentence_feature_object) 440 | 441 | universe_features = np.arange(0,1,0.001) 442 | universe_result = np.arange(0,1,0.01) 443 | 444 | title_word = ctrl.Antecedent(universe_features, 'title_word') 445 | sentence_length = ctrl.Antecedent(universe_features, 'sentence_length') 446 | sentence_location = ctrl.Antecedent(universe_features, 'sentence_location') 447 | numerical_data = ctrl.Antecedent(universe_features, 'numerical_data') 448 | # thematic_keyword = ctrl.Antecedent(universe_features, 'thematic_keyword') 449 | # proper_noun = ctrl.Antecedent(universe_features, 'proper_noun') 450 | # sentence_similarity = ctrl.Antecedent(universe_features, 'sentence_similarity') 451 | # term_weight = ctrl.Antecedent(universe_features, 'term_weight') 452 | 453 | 454 | result = ctrl.Consequent(universe_result, 'result') 455 | 456 | 457 | # Auto-membership function population is possible with .automf(3, 5, or 7) 458 | title_word.automf(3) 459 | sentence_length.automf(3) 460 | sentence_location.automf(3) 461 | numerical_data.automf(3) 462 | # thematic_keyword.automf(3) 463 | # proper_noun.automf(3) 464 | # sentence_similarity.automf(3) 465 | # term_weight.automf(3) 466 | 467 | # Custom membership functions can be built interactively with a familiar, 468 | # Pythonic API 469 | result['low'] = fuzzy.trimf(result.universe, [0.000,0.30,0.500]) 470 | result['medium'] = fuzzy.trimf(result.universe, [0.300,0.500,0.700]) 471 | result['high'] = fuzzy.trimf(result.universe, [0.500,1.0,1.0]) 472 | 473 | 474 | # 'poor'; 'average', or 'good' 475 | 476 | rule1 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 477 | rule2 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["average"], result['low']) 478 | rule3 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["good"], result['low']) 479 | rule4 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["poor"], result['low']) 480 | rule5 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["average"], result['low']) 481 | rule6 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["good"], result['low']) 482 | rule7 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["poor"], result['low']) 483 | rule8 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["average"], result['low']) 484 | rule9 = ctrl.Rule(title_word["poor"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["good"], result['low']) 485 | rule10 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 486 | rule11 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["average"], result['low']) 487 | rule12 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["good"], result['low']) 488 | rule13 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["average"] & numerical_data["poor"], result['low']) 489 | rule14 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["average"] & numerical_data["average"], result['medium']) 490 | rule15 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["average"] & numerical_data["good"], result['medium']) 491 | rule16 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["good"] & numerical_data["poor"], result['low']) 492 | rule17 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["good"] & numerical_data["average"], result['medium']) 493 | rule18 = ctrl.Rule(title_word["poor"] & sentence_length["average"] & sentence_location["good"] & numerical_data["good"], result['high']) 494 | rule19 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 495 | rule20 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["average"], result['low']) 496 | rule21 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["good"], result['low']) 497 | rule22 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["average"] & numerical_data["poor"], result['low']) 498 | rule23 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["average"] & numerical_data["average"], result['medium']) 499 | rule24 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["average"] & numerical_data["good"], result['high']) 500 | rule25 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["good"] & numerical_data["poor"], result['low']) 501 | rule26 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["good"] & numerical_data["average"], result['high']) 502 | rule27 = ctrl.Rule(title_word["poor"] & sentence_length["good"] & sentence_location["good"] & numerical_data["good"], result['high']) 503 | rule28 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 504 | rule29 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["average"], result['low']) 505 | rule30 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["good"], result['low']) 506 | rule31 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["poor"], result['low']) 507 | rule32 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["average"], result['medium']) 508 | rule33 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["good"], result['medium']) 509 | rule34 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["poor"], result['low']) 510 | rule35 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["average"], result['medium']) 511 | rule36 = ctrl.Rule(title_word["average"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["good"], result['high']) 512 | rule37 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 513 | rule38 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["average"], result['medium']) 514 | rule39 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["good"], result['medium']) 515 | rule40 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["average"] & numerical_data["poor"], result['medium']) 516 | rule41 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["average"] & numerical_data["average"], result['medium']) 517 | rule42 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["average"] & numerical_data["good"], result['medium']) 518 | rule43 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["good"] & numerical_data["poor"], result['medium']) 519 | rule44 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["good"] & numerical_data["average"], result['medium']) 520 | rule45 = ctrl.Rule(title_word["average"] & sentence_length["average"] & sentence_location["good"] & numerical_data["good"], result['medium']) 521 | rule46 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 522 | rule47 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["average"], result['medium']) 523 | rule48 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["good"], result['high']) 524 | rule49 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["average"] & numerical_data["poor"], result['medium']) 525 | rule50 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["average"] & numerical_data["average"], result['medium']) 526 | rule51 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["average"] & numerical_data["good"], result['medium']) 527 | rule52 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["good"] & numerical_data["poor"], result['high']) 528 | rule53 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["good"] & numerical_data["average"], result['medium']) 529 | rule54 = ctrl.Rule(title_word["average"] & sentence_length["good"] & sentence_location["good"] & numerical_data["good"], result['high']) 530 | rule55 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 531 | rule56 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["average"], result['low']) 532 | rule57 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["poor"] & numerical_data["good"], result['low']) 533 | rule58 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["poor"], result['low']) 534 | rule59 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["average"], result['medium']) 535 | rule60 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["average"] & numerical_data["good"], result['high']) 536 | rule61 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["poor"], result['low']) 537 | rule62 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["average"], result['high']) 538 | rule63 = ctrl.Rule(title_word["good"] & sentence_length["poor"] & sentence_location["good"] & numerical_data["good"], result['high']) 539 | rule64 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 540 | rule65 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["average"], result['medium']) 541 | rule66 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["poor"] & numerical_data["good"], result['high']) 542 | rule67 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["average"] & numerical_data["poor"], result['medium']) 543 | rule68 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["average"] & numerical_data["average"], result['medium']) 544 | rule69 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["average"] & numerical_data["good"], result['medium']) 545 | rule70 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["good"] & numerical_data["poor"], result['high']) 546 | rule71 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["good"] & numerical_data["average"], result['medium']) 547 | rule72 = ctrl.Rule(title_word["good"] & sentence_length["average"] & sentence_location["good"] & numerical_data["good"], result['high']) 548 | rule73 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["poor"], result['low']) 549 | rule74 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["average"], result['high']) 550 | rule75 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["poor"] & numerical_data["good"], result['high']) 551 | rule76 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["average"] & numerical_data["poor"], result['high']) 552 | rule77 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["average"] & numerical_data["average"], result['medium']) 553 | rule78 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["average"] & numerical_data["good"], result['high']) 554 | rule79 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["good"] & numerical_data["poor"], result['high']) 555 | rule80 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["good"] & numerical_data["average"], result['high']) 556 | rule81 = ctrl.Rule(title_word["good"] & sentence_length["good"] & sentence_location["good"] & numerical_data["good"], result['high']) 557 | 558 | tipping_ctrl = ctrl.ControlSystem([rule1, rule2, rule3, rule4, rule5, rule6, rule7, rule8, rule9, rule10, rule11, rule12, rule13, rule14, rule15, rule16, rule17, rule18, rule19, rule20, rule21, rule22, rule23, rule24, rule25, rule26, rule27, rule28, rule29, rule30, rule31, rule32, rule33, rule34, rule35, rule36, rule37, rule38, rule39, rule40, rule41, rule42, rule43, rule44, rule45, rule46, rule47, rule48, rule49, rule50, rule51, rule52, rule53, rule54, rule55, rule56, rule57, rule58, rule59, rule60, rule61, rule62, rule63, rule64, rule65, rule66, rule67, rule68, rule69, rule70, rule71, rule72, rule73, rule74, rule75, rule76, rule77, rule78, rule79, rule80, rule81]) 559 | # tipping_ctrl = acr.customRule() 560 | #tipping control simmulation 561 | tipping = ctrl.ControlSystemSimulation(tipping_ctrl) 562 | 563 | # Pass inputs to the ControlSystem using Antecedent labels with Pythonic API 564 | # Note: if you like passing many inputs all at once, use .inputs(dict_of_data) 565 | tipping.input['title_word'] = sentence_feature_object['title_word'] 566 | tipping.input['sentence_length'] = sentence_feature_object['sentence_length'] 567 | tipping.input['sentence_location'] = sentence_feature_object['sentence_location'] 568 | tipping.input['numerical_data'] = sentence_feature_object['numerical_data'] 569 | # tipping.input['thematic_keyword'] = sentence_feature_object['thematic_keyword'] 570 | # tipping.input['proper_noun'] = sentence_feature_object['proper_noun'] 571 | # tipping.input['sentence_similarity'] = sentence_feature_object['sentence_similarity'] 572 | # tipping.input['term_weight'] = sentence_feature_object['term_weight'] 573 | 574 | 575 | # tipping.inputs(sentence_feature_object) 576 | 577 | # Crunch the numbers 578 | tipping.compute() 579 | 580 | return tipping.output['result'] 581 | def starter (): 582 | raw_document_file = open("doc.txt","r") 583 | 584 | document_content = raw_document_file.read() 585 | 586 | text = document_content.split('\n', 1) 587 | 588 | sentences_list = [] 589 | Words = dict() 590 | 591 | title = text[0] 592 | text = text[1].replace(u"\u2018", '\'').replace(u"\u2019", '\'').replace(u"\u201c",'"').replace(u"\u201d", '"') 593 | 594 | title_text = title 595 | 596 | #preprocess title 597 | stemmer = nltk.stem.porter.PorterStemmer() 598 | 599 | #just remove stop words 600 | 601 | title_tokens = word_tokenize(title_text) 602 | title_tokens = [ stemmer.stem(token) for token in title_tokens if token not in stopwords_list] 603 | 604 | title_stemmed = ' '.join(title_tokens) 605 | 606 | sentence_detector = nltk.data.load('tokenizers/punkt/english.pickle') 607 | detected_sentences = sentence_detector.tokenize(text.strip()) 608 | 609 | #loop through the detected sentences 610 | # and take each sentence for preprocessing 611 | 612 | 613 | #print(detected_sentences) 614 | from nltk.stem import WordNetLemmatizer 615 | wordnet_lemmatizer = WordNetLemmatizer() 616 | # wordnet_lemmatizer.lemmatize(‘dogs’) 617 | 618 | for detected_sentence in detected_sentences : 619 | 620 | 621 | words_tokens = word_tokenize(detected_sentence) 622 | sentence = [wordnet_lemmatizer.lemmatize(word) for word in words_tokens if word not in stopwords_list] 623 | 624 | sentence = ' '.join(sentence) 625 | 626 | sentences_list.append(sentence) 627 | 628 | 629 | title_word_feature_value = title_word_feature(title_stemmed,sentences_list) 630 | sentence_length_feature_value = sentence_length_feature(sentences_list) 631 | sentence_location_feature_value = get_sentence_location_feature(sentences_list) 632 | numerical_data_feature_value = get_sentence_numerical_data(sentences_list) 633 | thematic_keyword_feature_value = thematic_keyword(title_stemmed, sentences_list) 634 | proper_noun_feature_value = proper_noun_feature(sentences_list) 635 | sentence_similarity_feature_value = sentence_to_sentence_similarity(sentences_list) 636 | term_weight_feature_value = termWeight(sentences_list) 637 | 638 | sentences_feature_list = [] 639 | 640 | fuzzy_list = [] 641 | 642 | sentence_total_object_result = [] 643 | 644 | indexCounter = 0 645 | 646 | for title_word,sentence_length,sentence_location, numerical_data, thematic_key, proper_noun,sentence_similarity,term_weight in zip(title_word_feature_value, sentence_length_feature_value, sentence_location_feature_value, numerical_data_feature_value, thematic_keyword_feature_value, proper_noun_feature_value,sentence_similarity_feature_value, term_weight_feature_value) : 647 | 648 | # custom 649 | #print(title_word,sentence_length,sentence_location, numerical_data, thematic_key, proper_noun,sentence_similarity,term_weight) 650 | 651 | sin_feature_obj = { 652 | 'title_word': title_word, 653 | 'sentence_length': sentence_length, 654 | 'sentence_location': sentence_location, 655 | 'numerical_data': numerical_data, 656 | 'thematic_keyword': thematic_key, 657 | 'proper_noun': proper_noun, 658 | 'sentence_similarity': sentence_similarity, 659 | 'term_weight': term_weight, 660 | } 661 | 662 | result = fuzzy_rules(sin_feature_obj) 663 | 664 | # result_object = {"sn" : indexCounter, "value" : result} 665 | # fuzzy_list.append(result_object) 666 | 667 | # custom 668 | #print(result) 669 | 670 | result_object = (indexCounter, result) 671 | fuzzy_list.append(result_object) 672 | 673 | indexCounter = indexCounter + 1 674 | 675 | 676 | # use 20% as compression ratio 677 | total_sentence = len(detected_sentences) 678 | compressionNumber = total_sentence * 0.20 679 | compressionNumber = int(compressionNumber) 680 | 681 | 682 | # sort the result according to the fuzzy result 683 | # and select up to compression rate 684 | result = sorted(fuzzy_list, key=lambda t: t[1], reverse=True)[:compressionNumber] 685 | result2 = sorted(result, key=lambda t: t[0]) 686 | 687 | 688 | for index in result2: 689 | print(detected_sentences[index[0]]) 690 | 691 | # for i in range(0, compressionNumber-1): 692 | # print(detected_sentences[fuzzy_list[i]['sn']]) 693 | 694 | 695 | starter() --------------------------------------------------------------------------------