├── Prodam
├── __init__.py
├── Common.py
├── AdmIndireta.py
├── Suspensas.py
├── Prodam.py
├── main.py
└── GabineteDoPrefeito.py
├── DiarioTools
├── __init__.py
├── GMailer.py
├── Log.py
├── Parser.py
├── Search.py
├── Retriever.py
├── ProdamMailer.py
├── Config.py
└── Process.py
├── ChefeDeGabinete
├── __init__.py
├── main.py
├── Nomeacao.py
├── Exoneracao.py
└── Substituicao.py
├── .gitignore
├── clean
├── Config
├── prodam.xml
├── chefesdegabinete.xml
└── config.xml.template
├── Backup.py
├── main.py
├── DlSanity.py
├── README.md
└── LICENSE
/Prodam/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/DiarioTools/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/ChefeDeGabinete/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.log
3 | *.pk
4 | *.swp
5 | *.orig
6 | *.htm
7 | *.html
8 | config*.xml
9 |
--------------------------------------------------------------------------------
/DiarioTools/GMailer.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LabProdam/LabDiario/HEAD/DiarioTools/GMailer.py
--------------------------------------------------------------------------------
/clean:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | rm -Rf `find . -name "*.pyc"`
3 | rm -Rf *.pk
4 | rm -Rf *.log
5 | rm -Rf *.html
6 |
--------------------------------------------------------------------------------
/Config/prodam.xml:
--------------------------------------------------------------------------------
1 |
2 | LabProdam - Atualizações - Prodam - Compras/Jurídico
3 | prodam.log
4 | Overwrite
5 |
6 |
--------------------------------------------------------------------------------
/Config/chefesdegabinete.xml:
--------------------------------------------------------------------------------
1 |
2 | LabProdam - Atualizações - Chefes de Gabinetes
3 | ChefesDeGabinete.log
4 | Overwrite
5 |
6 |
--------------------------------------------------------------------------------
/DiarioTools/Log.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 |
4 | import sys
5 | class Log(object):
6 | """Basic logging"""
7 | @staticmethod
8 | def Log(msg):
9 | print msg
10 | @staticmethod
11 | def Warning(msg):
12 | sys.stderr.write("WARNING: " + msg + "\n")
13 |
14 |
--------------------------------------------------------------------------------
/Backup.py:
--------------------------------------------------------------------------------
1 | import re
2 | import os
3 | import shutil
4 | import datetime
5 |
6 | def Backup():
7 | """Backup last execution pickles"""
8 | bkpFolder = "bkp"
9 | bkpName = os.path.join(bkpFolder,
10 | datetime.datetime.now().isoformat().replace("-", "").replace(":", "").replace(".", ""))
11 | if not os.path.exists(bkpFolder):
12 | os.mkdir(bkpFolder)
13 | os.mkdir(bkpName)
14 | files = os.listdir(".")
15 | for file in files:
16 | if re.search("\.pk", file):
17 | shutil.copy(file, bkpName)
18 |
--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from DiarioTools import Config
4 | from DiarioTools.Log import Log
5 | from DlSanity import *
6 | import sys
7 |
8 | import Backup
9 | Backup.Backup()
10 |
11 | def HandleSanity(configInstance):
12 | searcher = SearchSanity(configInstance, True)
13 | parser = ParseSanity()
14 | processor = ProcessorSanity(configInstance, searcher, parser, "SanityCheck.log", "SanityCheck")
15 | return processor.Process()
16 |
17 | cfg = Config.Configuration("Config/config.xml", sys.argv)
18 |
19 | if len(sys.argv) > 1 or HandleSanity(cfg):
20 | #Load and run modules present in config.xml
21 | for moduleName in cfg.modules:
22 | Log.Log("Importing :" + moduleName)
23 | module = __import__(moduleName, fromlist = ["main"])
24 | module.main.Run(moduleName + ".log")
25 | else:
26 | Log.Warning("Sanity Failed. Aborting")
27 |
28 |
--------------------------------------------------------------------------------
/Config/config.xml.template:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | LabProdam Mailer Robot -
7 |
8 |
9 |
10 |
11 |
13 | 2014/11/27
15 |
16 |
17 |
18 |
19 |
20 | Overwrite
21 |
22 |
23 | 20
24 | 3
25 | 1800
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/DiarioTools/Parser.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from Log import *
4 | import re
5 | class GenericParser(object):
6 | """ Given a set of regular expressions, evaluate them and convert groups
7 | of interest into array"""
8 | def __init__(self):
9 | self.expressions = []
10 | self.Initialize()
11 |
12 | def Initialize(self):
13 | """Override this method"""
14 | pass
15 |
16 | def AddExpression(self, reExpression, groupsOfInterest, flags = None, count = -1):
17 | self.expressions.append((reExpression, groupsOfInterest, flags, count))
18 |
19 |
20 | def Parse(self, content):
21 | for expression, groupsOfInterest, flags, count in self.expressions:
22 | if flags is not None:
23 | matches = re.finditer(expression, content, flags)
24 | else:
25 | matches = re.finditer(expression, content)
26 |
27 | yieldResult = False
28 | for num, match in enumerate(matches):
29 | if count >= 0 and num >= count:
30 | break
31 | matchGroups = []
32 | for group in groupsOfInterest:
33 | matchGroups.append(match.group(group))
34 | yield matchGroups
35 | yieldResult = True
36 |
37 | if not yieldResult:
38 | #yield empty response for keeping control when group is over
39 | yield []
40 |
41 |
--------------------------------------------------------------------------------
/DiarioTools/Search.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 |
4 | from DiarioTools.Log import *
5 | from DiarioTools.Retriever import *
6 | from DiarioTools.Parser import *
7 |
8 |
9 | class DlSearch(object):
10 | """Searcher module. Connects to colab and search the passed arguments"""
11 | def __init__(self, configuration, jsonFormat = False):
12 | self.configuration = configuration
13 | self.baseUrl = "http://devcolab.each.usp.br"
14 | self.options = {}
15 | self.query = None;
16 | self.queryAddr = "/do/?"
17 | if jsonFormat:
18 | self.queryAddr = "/do/catalog.json?"
19 |
20 | def SetDateOptions(self):
21 | if self.configuration.mode == "local search":
22 | self.options["date_range"] = "data:[" + self.configuration.startDate + " TO "+ self.configuration.endDate + "]"
23 | self.options["f[data][]"] = "date_range"
24 |
25 | def SetOptions(self):
26 | """ To be implemented by children"""
27 | pass
28 |
29 | def Search(self, query=None):
30 | """Searches accorgind to options set on SetOptions and query
31 | passed as argument or class attribute"""
32 | self.SetDateOptions()
33 | self.SetOptions()
34 | i = 0;
35 | while True:
36 | i += 1
37 | self.options["page"] = i
38 | if query is not None:
39 | self.options["q"] = query
40 | elif self.query is not None:
41 | self.options["q"] = self.query
42 | retriever = Retriever(self.baseUrl, self.queryAddr, self.options, self.configuration)
43 | contents = retriever.Retrieve()
44 | if contents is not None:
45 | yield contents
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
--------------------------------------------------------------------------------
/Prodam/Common.py:
--------------------------------------------------------------------------------
1 | #coding: utf-8
2 | import re
3 |
4 | class Tag(object):
5 | def __init__(self, startExpr, startReplace, endExpr, endReplace):
6 | self.se = "\(\(" + startExpr + "\)\)"
7 | self.sr = startReplace
8 | self.ee = "\(\(" + endExpr + "\)\)"
9 | self.er = endReplace
10 |
11 | def Apply(self, text):
12 | newText = text
13 | expressions = re.findall(self.se + ".*?" + self.ee, newText, re.I| re.S)
14 | for expression in expressions:
15 | newExpression = expression
16 | newExpression = re.sub(self.se, self.sr, newExpression, 1, re.I|re.S)
17 | newExpression = re.sub(self.ee, self.er, newExpression, 1, re.I|re.S)
18 | newText = newText.replace(expression, newExpression)
19 | return newText
20 |
21 | class DlTagsProcessor(object):
22 | def __init__(self, reOfInterest):
23 | bold = Tag("NG", "", "CL", "")
24 | self.reOfInterest = reOfInterest
25 | self.tags = [bold]
26 |
27 | def Process(self, text):
28 | parsedText = text
29 | for tag in self.tags:
30 | parsedText = tag.Apply(parsedText)
31 | parsedText = re.sub("\(\(T.TULO\)\)","", parsedText)
32 | parsedText = re.sub("\(\(TÍTULO\)\)","", parsedText)
33 | parsedText = re.sub("\(\(TEXTO\)\)","", parsedText)
34 | parsedText = re.sub("\(\(NG\)\)","", parsedText)
35 | parsedText = re.sub("\(\(CL\)\)","", parsedText)
36 | parsedText = re.sub("^\s*","", parsedText)
37 | parsedText = re.sub("^(.*)","
\\1
", parsedText)
38 |
39 |
40 | for line in parsedText.splitlines():
41 | for word in self.reOfInterest:
42 | if re.search(word, line, re.I) is not None:
43 | expression = re.search("[^\>]*" + word + "[^\<]*", line, re.I).group(0)
44 | parsedText = parsedText.replace(expression, "" + expression + "")
45 | break
46 | return parsedText
47 |
--------------------------------------------------------------------------------
/DiarioTools/Retriever.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from Log import *
4 | import urllib
5 | import socket
6 | import re
7 | import time
8 |
9 | class Retriever(object):
10 | """ Retrieves html contents from URL"""
11 | def __init__(self, baseUrl, queryAddr, options, configuration):
12 | self.baseUrl = baseUrl
13 | self.queryAddr = queryAddr
14 | self.options = options
15 |
16 | timeout = configuration.timeout
17 | socket.setdefaulttimeout(timeout)
18 |
19 | self.retries = configuration.retries
20 | self.timeBetweenRetries = configuration.timeBetweenRetries
21 |
22 | proxy = configuration.proxy
23 | if proxy is not None and len(proxy) > 0:
24 | self.proxy = {"http" : proxy}
25 | else:
26 | self.proxy = {}
27 |
28 | def Retrieve(self, retries = None, timeBetweenRetries = None):
29 | """Tries to fetch information from provided url
30 | retries is the number of attempts to acquire contents if timeout occurs
31 | timeBetweenRetries is the number of seconds to wait for the next attempt if a request times out
32 | """
33 | if retries is None:
34 | retries = self.retries
35 | if timeBetweenRetries is None:
36 | timeBetweenRetries = self.timeBetweenRetries
37 |
38 | contents = None
39 | url = self.baseUrl + self.queryAddr + urllib.urlencode(self.options)
40 | Log.Log("Searching: " + url)
41 | sd = None
42 | try:
43 | sd = urllib.urlopen(url, proxies = self.proxy)
44 | contents = sd.read()
45 | except IOError:
46 | retries -= 1
47 | if retries > 0:
48 | Log.Warning("TimedOut. Retrying more " + str(retries) + " times in " + str(timeBetweenRetries) + "s")
49 | time.sleep(timeBetweenRetries)
50 | self.Retrieve(retries)
51 | else:
52 | raise
53 | finally:
54 | if sd:
55 | sd.close()
56 | return contents;
57 |
58 |
59 |
60 |
61 |
--------------------------------------------------------------------------------
/ChefeDeGabinete/main.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from Nomeacao import *
4 | from Exoneracao import *
5 | from Substituicao import *
6 | from DiarioTools.Config import Configuration
7 | from DiarioTools.GMailer import *
8 | from DiarioTools.Log import *
9 | import datetime
10 | import sys
11 | import os
12 |
13 | logName = "Default.log"
14 |
15 | def HandleNomeacao(configInstance):
16 | searcher = SearchNomeacaoChefeDeGabinete(configInstance, True)
17 | parser = ParseNomeacaoChefeDeGabinete()
18 | processor = ProcessorNomeacaoChefeDeGabinete(configInstance, searcher, parser, logName, "NomeacaoChefeDeGabinete")
19 | return processor.Process()
20 |
21 | def HandleExoneracao(configInstance):
22 | searcher = SearchExoneracaoChefeDeGabinete(configInstance, True)
23 | parser = ParseExoneracaoChefeDeGabinete()
24 | processor = ProcessorExoneracaoChefeDeGabinete(configInstance, searcher, parser, logName, "ExoneracaoChefeDeGabinete")
25 | return processor.Process()
26 |
27 | def HandleSubstituicao(configInstance):
28 | searcher = SearchSubstituicaoChefeDeGabinete(configInstance, True)
29 | parser = ParseSubstituicaoChefeDeGabinete()
30 | processor = ProcessorSubstituicaoChefeDeGabinete(configInstance, searcher, parser, logName, "SubstituicaoChefeDeGabinete")
31 | return processor.Process()
32 |
33 | def Run(localLogName = "Default.log"):
34 | global logName
35 | logName = localLogName
36 |
37 | try:
38 | config = Configuration(os.path.join("Config","config.xml"), sys.argv, logName)
39 | config.AppendConfigurationFile(os.path.join("Config","chefesdegabinete.xml"))
40 | Log.Log("Searching Nomeacoes")
41 | messages = HandleNomeacao(config)
42 | Log.Log("Searching Exoneracoes")
43 | messages += HandleExoneracao(config)
44 | Log.Log("Searching Substituicoes")
45 | messages += HandleSubstituicao(config)
46 |
47 | if (config.mode == "alert mode"):
48 | messages = "Relatório de " + datetime.datetime.now().strftime("%d/%m/%y %H:%M:%S") + "\r\n\r\n" + messages
49 | Log.Log("Enviando E-Mail")
50 | mailer = GMailerWrappper(config)
51 | mailer.Send(messages)
52 | except Exception as e:
53 | Log.Warning("Problemas encontrados durante a execução do script")
54 | Log.Warning(str(e))
55 |
--------------------------------------------------------------------------------
/DlSanity.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from DiarioTools.Parser import *
4 | from DiarioTools.Process import *
5 | from DiarioTools.Search import *
6 | import re
7 | import datetime
8 |
9 | class ParseSanity(GenericParser):
10 | def Initialize(self):
11 | self.AddExpression(".*", [0], re.I|re.M)
12 |
13 | class SearchSanity(DlSearch):
14 | def SetOptions(self):
15 | self.options["sort"] = u"data desc"
16 | self.options["f[data][]"] = u"date_range"
17 |
18 | #Search if there has been an update in the last 7 days
19 | now = datetime.datetime.now()
20 | delta = datetime.timedelta(days=7)
21 | ref = now - delta
22 | endDate = "%04d-%02d-%02dT00:00:00.000Z" %(now.year, now.month, now.day)
23 | startDate = "%04d-%02d-%02dT00:00:00.000Z" %(ref.year, ref.month, ref.day)
24 | self.options["date_range"] = "data:[" + startDate + " TO "+ endDate + "]"
25 | self.options["f[data][]"] = "date_range"
26 |
27 | class ProcessorSanity(ResponseProcessor):
28 | def __init__(self, configInstance, searchObject, parseObject, fileName, sessionName):
29 | super(ProcessorSanity, self).__init__(configInstance, searchObject, parseObject, sessionName)
30 | self.newData = False
31 |
32 | def _ProcessIterate(self):
33 | for val in self.searchObject.Search():
34 | Log.Log("Iterating")
35 |
36 | parsedVal = json.loads(val)
37 | docs = parsedVal["response"]["docs"]
38 | if len(parsedVal["response"]["docs"]) == 0:
39 | break
40 |
41 | for doc in docs:
42 | self.doc = doc
43 | if self.configuration.mode == "local search":#Meaning start/end dates were not passed
44 | for response in self.parseObject.Parse(doc['texto']):
45 | self.Persist(response)
46 | return
47 |
48 | elif (self.lastSearch is not None and
49 | self.lastSearch.IsNewer(doc['id']) and
50 | IsNewer(doc['id'], self.configuration.baseDate)):
51 |
52 | self.lastSearch.SetCandidate(doc['id'])
53 | for response in self.parseObject.Parse(doc['texto']):
54 | self.Persist(response)
55 | return
56 |
57 | else:
58 | Log.Log("Iteration Over")
59 | return
60 |
61 | def Persist(self, data):
62 | """We just want to know if there is new data"""
63 | self.newData = True
64 |
65 | def ProcessEnd(self):
66 | return self.newData
67 |
--------------------------------------------------------------------------------
/ChefeDeGabinete/Nomeacao.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from DiarioTools.Parser import *
4 | from DiarioTools.Process import *
5 | from DiarioTools.Search import *
6 | import re
7 |
8 | class ParseNomeacaoChefeDeGabinete(GenericParser):
9 | def Initialize(self):
10 | self.AddExpression("^\s*Nomear.*?(senhora|senhor)\s*([^,]*).*?Chefe de Gabinete.(.*)", [2, 3, 0], re.I|re.M)
11 |
12 | class SearchNomeacaoChefeDeGabinete(DlSearch):
13 | def SetOptions(self):
14 | self.options["sort"] = u"data desc"
15 | self.query = "nomeação \"chefe de gabinete\""
16 |
17 | class ProcessorNomeacaoChefeDeGabinete(ResponseProcessor):
18 | def __init__(self, configInstance, searchObject, parseObject, fileName, sessionName):
19 | super(ProcessorNomeacaoChefeDeGabinete, self).__init__(configInstance, searchObject, parseObject, sessionName)
20 | self.fileName = fileName
21 | self.records = []
22 |
23 | with open(self.fileName, "a") as fd:
24 | fd.write("*** Nomeações ***\r\n")
25 |
26 | def Persist(self, data):
27 | if len(data) > 0:
28 | strOut = """Em """ + self.ProcessDate(data) + """, """ + self.ProcessName(data) + """ foi nomeado Chefe de Gabinete """ + self.ProcessGabinete(data) + "\n"
29 | self.records.append(strOut.encode("utf-8"))
30 | with open(self.fileName, "a") as fd:
31 | fd.write(strOut.encode("utf-8"))
32 |
33 | def ProcessEnd(self):
34 | message = "*** Nomeações ***\r\n"
35 | if (len(self.records) == 0):
36 | message += """Nenhum Chefe de Gabinete nomeado neste período\r\n\r\n"""
37 | Log.Log("Sem Alterações")
38 | else:
39 | message += "\r\n".join(self.records)
40 | message += "\r\n"
41 | return message
42 |
43 | def ProcessName(self, data):
44 | return data[0]
45 |
46 | def ProcessGabinete(self, data):
47 | gabineteRe = re.search("(Funda..o|Controladoria|Secretaria|Subprefeitura|Superintend.ncia)\s*,?\s*(([^\.](?! constante))*)", data[1], re.I)
48 | if gabineteRe is not None:
49 | gabineteFromData = gabineteRe.group(0)
50 | gabineteFromData = "da " + gabineteFromData
51 | else:
52 | gabineteRe = re.search("(Instituto|Servi.o)\s*,?\s*([^,]*)", data[1], re.I)
53 | if gabineteRe is not None:
54 | gabineteFromData = gabineteRe.group(0)
55 | gabineteFromData = "do " + gabineteFromData
56 | else:
57 | gabineteRe = re.search("^([^,]*).\s*s.mbolo", data[1], re.I)
58 | if gabineteRe is not None:
59 | gabineteFromData = gabineteRe.group(1)
60 | else:
61 | gabineteFromData = data[1]
62 | gabineteFromData = re.sub("s.mbolo \w*,", "", gabineteFromData, re.I)
63 | gabineteFromData = re.sub(",?\s*da Chefia de Gabinete[^,]*x", "", gabineteFromData, re.I)
64 | gabineteFromData = re.sub(",?\s*constante.*$", "", gabineteFromData, re.I)
65 | return gabineteFromData
66 |
67 | def ProcessDate(self, data):
68 | date = self.GetDateFromId()
69 | dateRe = re.search("a partir de ([^,]*)", data[2], re.I)
70 | if dateRe is not None:
71 | date = dateRe.group(1)
72 | return date
73 |
--------------------------------------------------------------------------------
/ChefeDeGabinete/Exoneracao.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from DiarioTools.Parser import *
4 | from DiarioTools.Process import *
5 | from DiarioTools.Search import *
6 | import re
7 |
8 | class ParseExoneracaoChefeDeGabinete(GenericParser):
9 | def Initialize(self):
10 | self.AddExpression("^\s*Exonerar.{0,1000}?(senhora|senhor)([^,]+).{0,400}?Chefe de Gabinete.(.+)", [2,3,0], re.I|re.M)
11 |
12 | class SearchExoneracaoChefeDeGabinete(DlSearch):
13 | def SetOptions(self):
14 | self.options["sort"] = u"data desc"
15 | self.query = "exonerar \"chefe de gabinete\""
16 |
17 | class ProcessorExoneracaoChefeDeGabinete(ResponseProcessor):
18 | def __init__(self, configInstance, searchObject, parseObject, fileName, sessionName):
19 | super(ProcessorExoneracaoChefeDeGabinete, self).__init__(configInstance, searchObject, parseObject, sessionName)
20 | self.fileName = fileName
21 | self.records = []
22 |
23 | with open(self.fileName, "a") as fd:
24 | fd.write("*** Exonerações ***\r\n")
25 |
26 | def Persist(self, data):
27 | if len(data) > 0:
28 | strOut = """Em """ + self.ProcessDate(data) + """, """ + self.ProcessName(data) + """ foi exonerado do cargo Chefe de Gabinete """ + self.ProcessGabinete(data) + "\n"
29 | self.records.append(strOut.encode("utf-8"))
30 | with open(self.fileName, "a") as fd:
31 | fd.write(strOut.encode("utf-8"))
32 |
33 | def ProcessEnd(self):
34 | message = "*** Exonerações ***\r\n"
35 | if (len(self.records) == 0):
36 | message += """Nenhum Chefe de Gabinete exonerado neste período\r\n\r\n"""
37 | Log.Log("Sem Alterações")
38 | else:
39 | message += "\r\n".join(self.records)
40 | message += "\r\n"
41 | return message
42 |
43 | def ProcessName(self, data):
44 | return data[0]
45 |
46 | def ProcessGabinete(self, data):
47 | gabineteRe = re.search("(Funda..o|Controladoria|Secretaria|Subprefeitura|Superintend.ncia)\s*,?\s*(([^\.](?! constante))*)", data[1], re.I)
48 | if gabineteRe is not None:
49 | gabineteFromData = gabineteRe.group(0)
50 | gabineteFromData = "da " + gabineteFromData
51 | else:
52 | gabineteRe = re.search("(Instituto|Servi.o)\s*,?\s*([^,]*)", data[1], re.I)
53 | if gabineteRe is not None:
54 | gabineteFromData = gabineteRe.group(0)
55 | gabineteFromData = "do " + gabineteFromData
56 | else:
57 | gabineteRe = re.search("^([^,]*).\s*s.mbolo", data[1], re.I)
58 | if gabineteRe is not None:
59 | gabineteFromData = gabineteRe.group(1)
60 | else:
61 | gabineteFromData = data[1]
62 | gabineteFromData = re.sub("s.mbolo \w*,", "", gabineteFromData, re.I)
63 | gabineteFromData = re.sub(",?\s*da Chefia de Gabinete[^,]*x", "", gabineteFromData, re.I)
64 | gabineteFromData = re.sub(",?\s*constante.*$", "", gabineteFromData, re.I)
65 | return gabineteFromData
66 |
67 | def ProcessDate(self, data):
68 | date = self.GetDateFromId()
69 | dateRe = re.search("a partir de ([^,]*)", data[2], re.I)
70 | if dateRe is not None:
71 | date = dateRe.group(1)
72 | return date
73 |
--------------------------------------------------------------------------------
/ChefeDeGabinete/Substituicao.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from DiarioTools.Parser import *
4 | from DiarioTools.Process import *
5 | from DiarioTools.Search import *
6 | import re
7 |
8 | class ParseSubstituicaoChefeDeGabinete(GenericParser):
9 | def Initialize(self):
10 | self.AddExpression("^.*?(senhora|senhor)([^,]+).{0,100}?Per.odo de ([^,]*).{0,100}?Substituir.{0,300}?(senhora|senhor)([^,]+).{0,300}?(chefe de gabinete.*)", [2,5,6,3], re.I|re.M)
11 |
12 | class SearchSubstituicaoChefeDeGabinete(DlSearch):
13 | def SetOptions(self):
14 | self.options["sort"] = u"data desc"
15 | self.query = "substituir \"chefe de gabinete\""
16 |
17 | class ProcessorSubstituicaoChefeDeGabinete(ResponseProcessor):
18 | def __init__(self, configInstance, searchObject, parseObject, fileName, sessionName):
19 | super(ProcessorSubstituicaoChefeDeGabinete, self).__init__(configInstance, searchObject, parseObject, sessionName)
20 | self.fileName = fileName
21 | self.records = []
22 |
23 | with open(self.fileName, "a") as fd:
24 | fd.write("*** Substituições ***\r\n")
25 |
26 | def Persist(self, data):
27 | if len(data) > 0:
28 | strOut = """Em """ + self.GetDateFromId() + """, """ + self.ProcessName1(data) + """ substitui """ + self.ProcessName2(data) + """, chefe de gabinete """ + self.ProcessGabinete(data) + """ de """ + self.ProcessPeriod(data)+ "\n"
29 | self.records.append(strOut.encode("utf-8"))
30 | with open(self.fileName, "a") as fd:
31 | fd.write(strOut.encode("utf-8"))
32 |
33 | def ProcessEnd(self):
34 | message = "*** Substituições ***\r\n"
35 | if (len(self.records) == 0):
36 | message += """Nenhum Chefe de Gabinete substituído neste período\r\n\r\n"""
37 | Log.Log("Sem Alterações")
38 | else:
39 | message += "\r\n".join(self.records)
40 | message += "\r\n"
41 | return message
42 |
43 | def ProcessName1(self, data):
44 | return data[0]
45 |
46 | def ProcessName2(self, data):
47 | return data[1]
48 |
49 | def ProcessPeriod(self, data):
50 | return data[3]
51 |
52 | def ProcessGabinete(self, data):
53 | gabineteRe = re.search("(Funda..o|Controladoria|Secretaria|Subprefeitura|Superintend.ncia)\s*,?\s*(([^\.](?! constante))*)", data[2], re.I)
54 | if gabineteRe is not None:
55 | gabineteFromData = gabineteRe.group(0)
56 | gabineteFromData = "da " + gabineteFromData
57 | else:
58 | gabineteRe = re.search("(Instituto|Servi.o)\s*,?\s*([^,]*)", data[2], re.I)
59 | if gabineteRe is not None:
60 | gabineteFromData = gabineteRe.group(0)
61 | gabineteFromData = "do " + gabineteFromData
62 | else:
63 | gabineteRe = re.search("^([^,]*).\s*s.mbolo", data[2], re.I)
64 | if gabineteRe is not None:
65 | gabineteFromData = gabineteRe.group(1)
66 | else:
67 | gabineteFromData = data[2]
68 | gabineteFromData = re.sub("s.mbolo \w*,", "", gabineteFromData, re.I)
69 | gabineteFromData = re.sub(",?\s*da Chefia de Gabinete[^,]*x", "", gabineteFromData, re.I)
70 | gabineteFromData = re.sub(",?\s*constante.*$", "", gabineteFromData, re.I)
71 | return gabineteFromData
72 |
--------------------------------------------------------------------------------
/DiarioTools/ProdamMailer.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from smtplib import *
4 | from Log import *
5 | from Config import Configuration, IfValidConfig
6 | from email.mime.text import MIMEText
7 | from email.mime.multipart import MIMEMultipart
8 | from email.header import Header
9 |
10 | import re
11 | import socket
12 | class ProxyException(Exception):
13 | def __repr__(self):
14 | return
15 | def __str__(self):
16 | return "Proxy Connection Error"
17 |
18 | class ProxiedSMTP(SMTP, object):
19 | """ Implement proxy layer above SMTP """
20 | def __init__(self, proxy = None, host="", port=0, local_hostname=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
21 | self.ProcessProxy(proxy)
22 | super(ProxiedSMTP, self).__init__(host, port, local_hostname, timeout)
23 |
24 | def ProcessProxy(self, proxy):
25 | self.proxyHost = None
26 | self.proxyPort = 0
27 |
28 | if proxy is not None:
29 | proxyRe = re.search("://([^:]*):(\d*)", proxy)
30 | if proxyRe is not None:
31 | self.proxyHost = socket.gethostbyname(proxyRe.group(1))
32 | self.proxyPort = int(proxyRe.group(2))
33 |
34 | def _get_socket(self, port, host, timeout):
35 | # This makes it simpler for SMTP_SSL to use the SMTP connect code
36 | # and just alter the socket connection bit.
37 | if self.proxyHost is None or len(self.proxyHost) <= 0:
38 | if self.debuglevel > 0:
39 | print>>stderr, 'connect:', (host, port)
40 | return socket.create_connection((port, host), timeout)
41 | else: #If proxy set
42 | # try:
43 | connSock = socket.create_connection((self.proxyHost, self.proxyPort), timeout)
44 | print "Connline: " + "CONNECT %s:%s HTTP/1.1\r\n\r\n" %(port, host)
45 | connSock.sendall("CONNECT %s:%s HTTP/1.1\r\n\r\n" %(port, host))
46 | rcv = connSock.recv(2048)
47 | print rcv
48 | result = re.search("\s(\d+)", rcv)
49 | if result:
50 | code = int(result.group(1))
51 | if code == 200:
52 | print "Connection successfull"
53 | return connSock
54 | else:
55 | raise ProxyException()
56 | # except:
57 | # raise ProxyException()
58 |
59 |
60 | class ProdamMailer(object):
61 | """ Sends e-mail through google server (only if valid configuration is found)"""
62 | def __init__(self, configInstance = None):
63 | if configInstance == None:
64 | Log.Warning("Configuração inválida. Não será possível mandar e-mails")
65 | else:
66 | self.config = configInstance
67 |
68 | @IfValidConfig
69 | def _PrepareMessage(self, messageText):
70 | multipart = MIMEMultipart('alternative')
71 | #multipart["From"] = Header(self.config.frommail.encode("utf-8"), "UTF-8").encode()
72 | #multipart["To"] = Header("; ".join(self.config.destination).encode("utf-8"), "UTF-8").encode()
73 | multipart["Subject"] = Header(self.config.subject.encode("utf-8"), "UTF-8").encode()
74 |
75 | body = self.config.header + "\r\n\r\n"
76 | body += messageText.decode("utf-8")
77 | body += "\r\n\r\n" + self.config.footer
78 | multipart.attach(MIMEText(body.encode("utf-8"), 'plain', 'UTF-8'))
79 | return multipart.as_string()
80 |
81 | @IfValidConfig
82 | def Send(self, messageText):
83 | message = self._PrepareMessage(messageText)
84 | try:
85 | server = ProxiedSMTP(self.config.proxy, self.config.serverAddr, self.config.serverPort)
86 | server.starttls()
87 | server.login(self.config.username, self.config.password)
88 | server.sendmail(self.config.frommail, self.config.destination, message)
89 | server.quit()
90 | Log.Log("E-mail enviado")
91 | except:
92 | Log.Warning("Não foi possível enviar e-mails")
93 |
94 |
--------------------------------------------------------------------------------
/Prodam/AdmIndireta.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from Common import *
4 | from DiarioTools.Parser import *
5 | from DiarioTools.Process import *
6 | from DiarioTools.Search import *
7 | import re
8 |
9 | wordsOfInterest = ["Administração Indireta"]
10 | reOfInterest = ["Prodam", "Administra..o Indireta"]
11 |
12 | class ParseAdmIndireta(GenericParser):
13 | def Initialize(self):
14 | self.AddExpression(".+", [0], re.I|re.S)
15 |
16 | class SearchAdmIndireta(DlSearch):
17 | global wordsOfInterest
18 | def SetOptions(self):
19 | self.options["sort"] = u"data desc"
20 |
21 | query = ""
22 | for word in wordsOfInterest:
23 | query += "\"" + word + "\" "
24 | self.query = query
25 |
26 | self.options["f[tipo_conteudo_facet][]"] = "DESPACHO"
27 |
28 | class ProcessorAdmIndireta(ResponseProcessor):
29 | def __init__(self, configInstance, searchObject, parseObject, fileName, sessionName):
30 | super(ProcessorAdmIndireta, self).__init__(configInstance, searchObject, parseObject, sessionName)
31 | self.fileName = fileName
32 | self.data = ""
33 | self.atLeadOneFound = False
34 | self.dlProcessor = DlTagsProcessor(reOfInterest)
35 |
36 | with open(self.fileName, "a") as fd:
37 | fd.write("*** Administração indireta ***\r\n")
38 | self.data += """
39 |
40 |
41 |
42 |
43 |
44 |
92 | """
93 |
94 | def Persist(self, data):
95 | self.atLeadOneFound = True
96 | contents = data[0].encode("utf-8")
97 | with open(self.fileName, "a") as fd:
98 | fd.write(contents)
99 |
100 | self.data += ""
101 | self.data += """
102 |
103 |
[Esconder]
104 |
110 | """ + self.dlProcessor.Process(contents) + """
\n\n"""
111 | self.data += "
"
112 |
113 | def ProcessEnd(self):
114 | if not self.atLeadOneFound:
115 | return None
116 | else:
117 | self.data += ""
118 | return self.data
119 |
120 |
--------------------------------------------------------------------------------
/Prodam/Suspensas.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | #coding: utf-8
3 | from Common import *
4 | from DiarioTools.Parser import *
5 | from DiarioTools.Process import *
6 | from DiarioTools.Search import *
7 | import re
8 |
9 | wordsOfInterest = ["SUSPENSAS DE PARTICIPAÇÃO EM LICITAÇÃO E IMPEDIDAS DE CONTRATAR COM A ADMINISTRAÇÃO"
10 | ]
11 |
12 | reOfInterest = ["Prodam", "SUSPENSAS DE PARTICIPA..O EM LICITA..O E IMPEDIDAS DE CONTRATAR COM A ADMINISTRA..O"
13 | ]
14 |
15 | class ParseSuspensas(GenericParser):
16 | def Initialize(self):
17 | self.AddExpression(".+", [0], re.I|re.S)
18 |
19 | class SearchSuspensas(DlSearch):
20 | global wordsOfInterest
21 | def SetOptions(self):
22 | self.options["sort"] = u"data desc"
23 |
24 | query = ""
25 | for word in wordsOfInterest:
26 | query += "\"" + word + "\" "
27 | self.query = query
28 |
29 | class ProcessorSuspensas(ResponseProcessor):
30 | def __init__(self, configInstance, searchObject, parseObject, fileName, sessionName):
31 | super(ProcessorSuspensas, self).__init__(configInstance, searchObject, parseObject, sessionName)
32 | self.fileName = fileName
33 | self.data = ""
34 | self.atLeadOneFound = False
35 | self.dlProcessor = DlTagsProcessor(reOfInterest)
36 |
37 | with open(self.fileName, "a") as fd:
38 | fd.write("*** Suspensas ***\r\n")
39 | self.data += """
40 |
41 |
42 |
43 |
44 |
45 |
93 | """
94 |
95 | def Persist(self, data):
96 | self.atLeadOneFound = True
97 | contents =data[0].encode("utf-8")
98 | with open(self.fileName, "a") as fd:
99 | fd.write(contents)
100 |
101 | self.data += ""
102 | self.data += """
103 |
104 |
[Esconder]
105 |
111 | """ + self.dlProcessor.Process(contents) + """
\n\n"""
112 | self.data += "
"
113 |
114 | def ProcessEnd(self):
115 | self.data += ""
116 | return self.data
117 |
118 | def ProcessEnd(self):
119 | if not self.atLeadOneFound:
120 | return None
121 | else:
122 | self.data += "