├── .gitignore
├── LICENSE
├── README.md
├── dbx_Pg.pl
├── dbx_mysql.pl
├── dmarcts-report-parser.conf.sample
└── dmarcts-report-parser.pl
/.gitignore:
--------------------------------------------------------------------------------
1 | /dmarcts-report-parser.conf
2 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | GNU GENERAL PUBLIC LICENSE
2 | Version 3, 29 June 2007
3 |
4 | Copyright (C) 2007 Free Software Foundation, Inc.
5 | Everyone is permitted to copy and distribute verbatim copies
6 | of this license document, but changing it is not allowed.
7 |
8 | Preamble
9 |
10 | The GNU General Public License is a free, copyleft license for
11 | software and other kinds of works.
12 |
13 | The licenses for most software and other practical works are designed
14 | to take away your freedom to share and change the works. By contrast,
15 | the GNU General Public License is intended to guarantee your freedom to
16 | share and change all versions of a program--to make sure it remains free
17 | software for all its users. We, the Free Software Foundation, use the
18 | GNU General Public License for most of our software; it applies also to
19 | any other work released this way by its authors. You can apply it to
20 | your programs, too.
21 |
22 | When we speak of free software, we are referring to freedom, not
23 | price. Our General Public Licenses are designed to make sure that you
24 | have the freedom to distribute copies of free software (and charge for
25 | them if you wish), that you receive source code or can get it if you
26 | want it, that you can change the software or use pieces of it in new
27 | free programs, and that you know you can do these things.
28 |
29 | To protect your rights, we need to prevent others from denying you
30 | these rights or asking you to surrender the rights. Therefore, you have
31 | certain responsibilities if you distribute copies of the software, or if
32 | you modify it: responsibilities to respect the freedom of others.
33 |
34 | For example, if you distribute copies of such a program, whether
35 | gratis or for a fee, you must pass on to the recipients the same
36 | freedoms that you received. You must make sure that they, too, receive
37 | or can get the source code. And you must show them these terms so they
38 | know their rights.
39 |
40 | Developers that use the GNU GPL protect your rights with two steps:
41 | (1) assert copyright on the software, and (2) offer you this License
42 | giving you legal permission to copy, distribute and/or modify it.
43 |
44 | For the developers' and authors' protection, the GPL clearly explains
45 | that there is no warranty for this free software. For both users' and
46 | authors' sake, the GPL requires that modified versions be marked as
47 | changed, so that their problems will not be attributed erroneously to
48 | authors of previous versions.
49 |
50 | Some devices are designed to deny users access to install or run
51 | modified versions of the software inside them, although the manufacturer
52 | can do so. This is fundamentally incompatible with the aim of
53 | protecting users' freedom to change the software. The systematic
54 | pattern of such abuse occurs in the area of products for individuals to
55 | use, which is precisely where it is most unacceptable. Therefore, we
56 | have designed this version of the GPL to prohibit the practice for those
57 | products. If such problems arise substantially in other domains, we
58 | stand ready to extend this provision to those domains in future versions
59 | of the GPL, as needed to protect the freedom of users.
60 |
61 | Finally, every program is threatened constantly by software patents.
62 | States should not allow patents to restrict development and use of
63 | software on general-purpose computers, but in those that do, we wish to
64 | avoid the special danger that patents applied to a free program could
65 | make it effectively proprietary. To prevent this, the GPL assures that
66 | patents cannot be used to render the program non-free.
67 |
68 | The precise terms and conditions for copying, distribution and
69 | modification follow.
70 |
71 | TERMS AND CONDITIONS
72 |
73 | 0. Definitions.
74 |
75 | "This License" refers to version 3 of the GNU General Public License.
76 |
77 | "Copyright" also means copyright-like laws that apply to other kinds of
78 | works, such as semiconductor masks.
79 |
80 | "The Program" refers to any copyrightable work licensed under this
81 | License. Each licensee is addressed as "you". "Licensees" and
82 | "recipients" may be individuals or organizations.
83 |
84 | To "modify" a work means to copy from or adapt all or part of the work
85 | in a fashion requiring copyright permission, other than the making of an
86 | exact copy. The resulting work is called a "modified version" of the
87 | earlier work or a work "based on" the earlier work.
88 |
89 | A "covered work" means either the unmodified Program or a work based
90 | on the Program.
91 |
92 | To "propagate" a work means to do anything with it that, without
93 | permission, would make you directly or secondarily liable for
94 | infringement under applicable copyright law, except executing it on a
95 | computer or modifying a private copy. Propagation includes copying,
96 | distribution (with or without modification), making available to the
97 | public, and in some countries other activities as well.
98 |
99 | To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies. Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 |
103 | An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License. If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 |
112 | 1. Source Code.
113 |
114 | The "source code" for a work means the preferred form of the work
115 | for making modifications to it. "Object code" means any non-source
116 | form of a work.
117 |
118 | A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 |
123 | The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form. A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 |
134 | The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities. However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work. For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 |
147 | The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 |
151 | The Corresponding Source for a work in source code form is that
152 | same work.
153 |
154 | 2. Basic Permissions.
155 |
156 | All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met. This License explicitly affirms your unlimited
159 | permission to run the unmodified Program. The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work. This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 |
164 | You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force. You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright. Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 |
175 | Conveying under any other circumstances is permitted solely under
176 | the conditions stated below. Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 |
179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 |
181 | No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 |
187 | When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 |
195 | 4. Conveying Verbatim Copies.
196 |
197 | You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 |
205 | You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 |
208 | 5. Conveying Modified Source Versions.
209 |
210 | You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 |
214 | a) The work must carry prominent notices stating that you modified
215 | it, and giving a relevant date.
216 |
217 | b) The work must carry prominent notices stating that it is
218 | released under this License and any conditions added under section
219 | 7. This requirement modifies the requirement in section 4 to
220 | "keep intact all notices".
221 |
222 | c) You must license the entire work, as a whole, under this
223 | License to anyone who comes into possession of a copy. This
224 | License will therefore apply, along with any applicable section 7
225 | additional terms, to the whole of the work, and all its parts,
226 | regardless of how they are packaged. This License gives no
227 | permission to license the work in any other way, but it does not
228 | invalidate such permission if you have separately received it.
229 |
230 | d) If the work has interactive user interfaces, each must display
231 | Appropriate Legal Notices; however, if the Program has interactive
232 | interfaces that do not display Appropriate Legal Notices, your
233 | work need not make them do so.
234 |
235 | A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit. Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 |
245 | 6. Conveying Non-Source Forms.
246 |
247 | You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 |
252 | a) Convey the object code in, or embodied in, a physical product
253 | (including a physical distribution medium), accompanied by the
254 | Corresponding Source fixed on a durable physical medium
255 | customarily used for software interchange.
256 |
257 | b) Convey the object code in, or embodied in, a physical product
258 | (including a physical distribution medium), accompanied by a
259 | written offer, valid for at least three years and valid for as
260 | long as you offer spare parts or customer support for that product
261 | model, to give anyone who possesses the object code either (1) a
262 | copy of the Corresponding Source for all the software in the
263 | product that is covered by this License, on a durable physical
264 | medium customarily used for software interchange, for a price no
265 | more than your reasonable cost of physically performing this
266 | conveying of source, or (2) access to copy the
267 | Corresponding Source from a network server at no charge.
268 |
269 | c) Convey individual copies of the object code with a copy of the
270 | written offer to provide the Corresponding Source. This
271 | alternative is allowed only occasionally and noncommercially, and
272 | only if you received the object code with such an offer, in accord
273 | with subsection 6b.
274 |
275 | d) Convey the object code by offering access from a designated
276 | place (gratis or for a charge), and offer equivalent access to the
277 | Corresponding Source in the same way through the same place at no
278 | further charge. You need not require recipients to copy the
279 | Corresponding Source along with the object code. If the place to
280 | copy the object code is a network server, the Corresponding Source
281 | may be on a different server (operated by you or a third party)
282 | that supports equivalent copying facilities, provided you maintain
283 | clear directions next to the object code saying where to find the
284 | Corresponding Source. Regardless of what server hosts the
285 | Corresponding Source, you remain obligated to ensure that it is
286 | available for as long as needed to satisfy these requirements.
287 |
288 | e) Convey the object code using peer-to-peer transmission, provided
289 | you inform other peers where the object code and Corresponding
290 | Source of the work are being offered to the general public at no
291 | charge under subsection 6d.
292 |
293 | A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 |
297 | A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling. In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage. For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product. A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 |
310 | "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source. The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 |
318 | If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information. But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 |
329 | The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed. Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 |
337 | Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 |
343 | 7. Additional Terms.
344 |
345 | "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law. If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 |
354 | When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it. (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.) You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 |
361 | Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 |
365 | a) Disclaiming warranty or limiting liability differently from the
366 | terms of sections 15 and 16 of this License; or
367 |
368 | b) Requiring preservation of specified reasonable legal notices or
369 | author attributions in that material or in the Appropriate Legal
370 | Notices displayed by works containing it; or
371 |
372 | c) Prohibiting misrepresentation of the origin of that material, or
373 | requiring that modified versions of such material be marked in
374 | reasonable ways as different from the original version; or
375 |
376 | d) Limiting the use for publicity purposes of names of licensors or
377 | authors of the material; or
378 |
379 | e) Declining to grant rights under trademark law for use of some
380 | trade names, trademarks, or service marks; or
381 |
382 | f) Requiring indemnification of licensors and authors of that
383 | material by anyone who conveys the material (or modified versions of
384 | it) with contractual assumptions of liability to the recipient, for
385 | any liability that these contractual assumptions directly impose on
386 | those licensors and authors.
387 |
388 | All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10. If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term. If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 |
398 | If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 |
403 | Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 |
407 | 8. Termination.
408 |
409 | You may not propagate or modify a covered work except as expressly
410 | provided under this License. Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 |
415 | However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 |
422 | Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 |
429 | Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License. If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 |
435 | 9. Acceptance Not Required for Having Copies.
436 |
437 | You are not required to accept this License in order to receive or
438 | run a copy of the Program. Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance. However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work. These actions infringe copyright if you do
443 | not accept this License. Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 |
446 | 10. Automatic Licensing of Downstream Recipients.
447 |
448 | Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License. You are not responsible
451 | for enforcing compliance by third parties with this License.
452 |
453 | An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations. If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 |
463 | You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License. For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 |
471 | 11. Patents.
472 |
473 | A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based. The
475 | work thus licensed is called the contributor's "contributor version".
476 |
477 | A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version. For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 |
487 | Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 |
492 | In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement). To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 |
499 | If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients. "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 |
513 | If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 |
521 | A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License. You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 |
536 | Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 |
540 | 12. No Surrender of Others' Freedom.
541 |
542 | If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License. If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all. For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 |
552 | 13. Use with the GNU Affero General Public License.
553 |
554 | Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work. The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 |
563 | 14. Revised Versions of this License.
564 |
565 | The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time. Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 |
570 | Each version is given a distinguishing version number. If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation. If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 |
579 | If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 |
584 | Later license versions may give you additional or different
585 | permissions. However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 |
589 | 15. Disclaimer of Warranty.
590 |
591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 |
600 | 16. Limitation of Liability.
601 |
602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 |
612 | 17. Interpretation of Sections 15 and 16.
613 |
614 | If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 |
621 | END OF TERMS AND CONDITIONS
622 |
623 | How to Apply These Terms to Your New Programs
624 |
625 | If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 |
629 | To do so, attach the following notices to the program. It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 |
634 | {one line to give the program's name and a brief idea of what it does.}
635 | Copyright (C) {year} {name of author}
636 |
637 | This program is free software: you can redistribute it and/or modify
638 | it under the terms of the GNU General Public License as published by
639 | the Free Software Foundation, either version 3 of the License, or
640 | (at your option) any later version.
641 |
642 | This program is distributed in the hope that it will be useful,
643 | but WITHOUT ANY WARRANTY; without even the implied warranty of
644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
645 | GNU General Public License for more details.
646 |
647 | You should have received a copy of the GNU General Public License
648 | along with this program. If not, see .
649 |
650 | Also add information on how to contact you by electronic and paper mail.
651 |
652 | If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 |
655 | {project} Copyright (C) {year} {fullname}
656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 | This is free software, and you are welcome to redistribute it
658 | under certain conditions; type `show c' for details.
659 |
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License. Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 |
664 | You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | .
668 |
669 | The GNU General Public License does not permit incorporating your program
670 | into proprietary programs. If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library. If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License. But first, please read
674 | .
675 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # dmarcts-report-parser
2 | A Perl based tool to parse DMARC reports, based on John Levine's [rddmarc](http://www.taugh.com/rddmarc/), but extended by the following features:
3 | * Allow to read messages from an IMAP server and not only from the local filesystem.
4 | * Store much more XML values into the database (for example the missing SPF and DKIM results from the policy_evaluated section) and also the entire XML for later reference.
5 | * Supports MySQL and PostgreSQL.
6 | * Needed database tables and columns are created automatically, user only needs to provide a database. The database schema is compatible to the one used by rddmarc, but extends it by additional fields. Users can switch from rddmarc to dmarcts-report-parser without having to do any changes to the database by themselves.
7 | * Due to limitations in stock configurations of MySQL/MariaSQL on some distros, it may be necessary
8 | to add the following to your configuration (i.e. in /etc/mysql/mariadb.conf.d/50-server.cnf):
9 |
10 | ```
11 | innodb_large_prefix = on
12 | innodb_file_format = barracuda
13 | innodb_file_per_table = true
14 | ```
15 |
16 | ## Installation and Configuration
17 |
18 | To install dependencies...
19 |
20 | ### on Debian:
21 | ```
22 | apt-get install libfile-mimeinfo-perl libmail-imapclient-perl libmime-tools-perl libxml-simple-perl \
23 | libio-socket-inet6-perl libio-socket-ip-perl libperlio-gzip-perl \
24 | libmail-mbox-messageparser-perl unzip
25 | ```
26 | Plus `libdbd-mysql-perl` for MySQL or `libdbd-pg-perl` for PostgreSQL.
27 | ### on Fedora (Fedora 23):
28 | ```
29 | sudo dnf install perl-File-MimeInfo perl-Mail-IMAPClient perl-MIME-tools perl-XML-Simple perl-DBI \
30 | perl-Socket6 perl-PerlIO-gzip unzip
31 | ```
32 | Plus `perl-DBD-MySQL` for MySQL or `perl-DBD-Pg` for PostgreSQL.
33 | ### on CentOS (CentOS 7):
34 | ```
35 | yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
36 | yum install perl-File-MimeInfo perl-Mail-IMAPClient perl-MIME-tools perl-XML-Simple perl-DBI \
37 | perl-Socket6 perl-PerlIO-gzip unzip perl-Mail-Mbox-MessageParser
38 | ```
39 | Plus `perl-DBD-MySQL` for MySQL or `perl-DBD-Pg` for PostgreSQL.
40 | ### on FreeBSD (FreeBSD 13.2):
41 | ```
42 | sudo pkg install p5-File-MimeInfo p5-Mail-IMAPClient p5-MIME-tools p5-XML-Simple p5-DBI p5-Socket6 p5-PerlIO-gzip p5-Mail-Mbox-MessageParser unzip
43 | ```
44 | Plus `p5-DBD-MySQL` for MySQL or `p5-DBD-Pg` for PostgreSQL.
45 |
46 | There is also a port in mail/dmarcts-report-parser that does all this and
47 | creates a periodic script that runs the IMAP pull daily:
48 | ```
49 | sudo pkg install dmarcts-report-parser
50 | [create the database and IMAP users and edit /usr/local/etc/dmarcts-report-parser.conf]
51 | echo daily_dmarcts_report_parser_enable=yes >> /etc/periodic.conf
52 | # Test it
53 | dmarcts-report-parser.pl -i
54 | # Hide the passwords!
55 | chmod 600 /usr/local/etc/dmarcts-report-parser.conf
56 | ```
57 | ### on macOS (macOS 10.13):
58 | ```
59 | brew install mysql shared-mime-info
60 | update-mime-database /usr/local/share/mime
61 | perl -MCPAN -e 'install Mail::IMAPClient'
62 | perl -MCPAN -e 'install Mail::Mbox::MessageParser'
63 | perl -MCPAN -e 'install File::MimeInfo'
64 | ```
65 | Plus `perl -MCPAN -e 'install DBD::mysql'` por MySQL or `perl -MCPAN -e 'install DBD::Pg'` or PostgreSQL.
66 |
67 | To get your copy of the dmarcts-report-parser, you can either clone the repository:
68 | ```
69 | git clone https://github.com/techsneeze/dmarcts-report-parser.git
70 | ```
71 | or download a zip file containg all files from [here](https://github.com/techsneeze/dmarcts-report-parser/archive/master.zip). Once the files have been downloaded, you will need to copy/rename `dmarcts-report-parser.conf.sample` to `dmarcts-report-parser.conf`. Next, edit the configuration options:
72 |
73 | ```
74 | ################################################################################
75 | ### configuration ##############################################################
76 | ################################################################################
77 |
78 | # If IMAP access is not used, config options starting with $imap do not need to
79 | # be set and are ignored.
80 |
81 | $debug = 0;
82 | $delete_reports = 0;
83 |
84 | # Supported types: mysql, Pg. If unset, defaults to mysql
85 | #$dbtype = 'mysql';
86 | $dbname = 'dmarc';
87 | $dbuser = 'dmarc';
88 | $dbpass = 'password';
89 | $dbhost = 'dbhost'; # Set the hostname if we can't connect to the local socket.
90 | $dbport = '3306';
91 |
92 | $imapserver = 'imap.server';
93 | $imapuser = 'username';
94 | $imappass = 'password';
95 | $imapport = '143';
96 | $imapssl = '0'; # If set to 1, remember to change server port to 993 and disable imaptls.
97 | $imaptls = '0'; # Enabled as the default and best-practice.
98 | $tlsverify = '0'; # Enable verify server cert as the default and best-practice.
99 | $imapignoreerror = '0'; # set it to 1 if you see an "ERROR: message_string()
100 | # expected 119613 bytes but received 81873 you may
101 | # need the IgnoreSizeErrors option" because of malfunction
102 | # imap server as MS Exchange 2007, ...
103 | $imapreadfolder = 'dmarc';
104 |
105 | # If $imapmovefolder is set, processed IMAP messages will be moved (overruled by
106 | # the --delete option!)
107 | $imapmovefolder = 'dmarc/processed';
108 |
109 | # maximum size of XML files to store in database, long files can cause transaction aborts
110 | $maxsize_xml = 50000;
111 | # store XML as base64 encopded gzip in database (save space, harder usable)
112 | $compress_xml = 0;
113 |
114 | # if there was an error during file processing (message does not contain XML or ZIP parts,
115 | # or a database error) the parser reports an error and does not delete the file, even if
116 | # delete_reports is set (or --delete is given). Deletion can be enforced by delete_failed,
117 | # however not for database errors.
118 | $delete_failed = 0;
119 | ```
120 | The script is looking for `dmarcts-report-parser.conf` in the current working directory. If not found it will look by the calling path. If neither is found than it will abort.
121 |
122 | Note: Be sure to use the proper hierarchy separator for your server in all folder specs, and
123 | if your IMAP server flattens the hierarchy (i.e. Cyrus IMAP with "altnamespace: yes") then
124 | leave "Inbox" off of the beginning of such specs.
125 |
126 | ## Usage
127 |
128 | ```
129 | ./dmarcts-report-parser.pl [OPTIONS] [PATH]
130 | ```
131 | PATH can be the filename of a single file or a list of files - wildcard expression are allowed.
132 |
133 | **Remember**: This script needs a configurations file called in the current working directory, which defines a database server with credentials and (if used) an IMAP server with credentials.
134 |
135 | One of the following source options must be provided:
136 | ```
137 | # -i : Read reports from messages on IMAP server as defined in the config file.
138 | # -m : Read reports from mbox file(s) provided in PATH.
139 | # -e : Read reports from MIME email file(s) provided in PATH.
140 | # -x : Read reports from xml file(s) provided in PATH.
141 | # -z : Read reports from zip file(s) provided in PATH.
142 | ```
143 |
144 | The following options are always allowed:
145 | ```
146 | # -d : Print debug info.
147 | # -r : Replace existing reports rather than failing.
148 | # --delete : Delete processed message files (the XML is stored in the
149 | # database for later reference).
150 | ```
151 |
152 | More info can currently be found at : [TechSneeze.com](http://www.techsneeze.com/how-parse-dmarc-reports-imap/)
153 |
--------------------------------------------------------------------------------
/dbx_Pg.pl:
--------------------------------------------------------------------------------
1 | %dbx = (
2 | epoch_to_timestamp_fn => 'TO_TIMESTAMP',
3 |
4 | to_hex_string => sub {
5 | my ($bin) = @_;
6 | return "'\\x" . unpack("H*", $bin) . "'";
7 | },
8 |
9 | column_info_type_col => 'pg_type',
10 |
11 | tables => {
12 | "report" => {
13 | column_definitions => [
14 | "serial" , "bigint" , "GENERATED ALWAYS AS IDENTITY",
15 | "mindate" , "timestamp without time zone" , "NOT NULL",
16 | "maxdate" , "timestamp without time zone" , "NULL",
17 | "domain" , "character varying(255)" , "NOT NULL",
18 | "org" , "character varying(255)" , "NOT NULL",
19 | "reportid" , "character varying(255)" , "NOT NULL",
20 | "email" , "character varying(255)" , "NULL",
21 | "extra_contact_info" , "character varying(255)" , "NULL",
22 | "policy_adkim" , "character varying(20)" , "NULL",
23 | "policy_aspf" , "character varying(20)" , "NULL",
24 | "policy_p" , "character varying(20)" , "NULL",
25 | "policy_sp" , "character varying(20)" , "NULL",
26 | "policy_pct" , "smallint" , "",
27 | "raw_xml" , "text" , "",
28 | ],
29 | additional_definitions => "PRIMARY KEY (serial)",
30 | table_options => "",
31 | indexes => [
32 | "CREATE UNIQUE INDEX report_uidx_domain ON report (domain, org, reportid);"
33 | ],
34 | },
35 | "rptrecord" => {
36 | column_definitions => [
37 | "id" , "bigint" , "GENERATED ALWAYS AS IDENTITY",
38 | "serial" , "bigint" , "NOT NULL",
39 | "ip" , "bigint" , "",
40 | "ip6" , "bytea" , "",
41 | "rcount" , "integer" , "NOT NULL",
42 | "disposition" , "character varying(20)" , "",
43 | "reason" , "character varying(255)" , "",
44 | "dkimdomain" , "character varying(255)" , "",
45 | "dkimresult" , "character varying(20)" , "",
46 | "spfdomain" , "character varying(255)" , "",
47 | "spfresult" , "character varying(20)" , "",
48 | "spf_align" , "character varying(20)" , "NOT NULL",
49 | "dkim_align" , "character varying(20)" , "NOT NULL",
50 | "identifier_hfrom" , "character varying(255)" , ""
51 | ],
52 | additional_definitions => "PRIMARY KEY (id)",
53 | table_options => "",
54 | indexes => [
55 | "CREATE INDEX rptrecord_idx_serial ON rptrecord (serial, ip);",
56 | "CREATE INDEX rptrecord_idx_serial6 ON rptrecord (serial, ip6);",
57 | ],
58 | },
59 | },
60 |
61 | add_column => sub {
62 | my ($table, $col_name, $col_type, $col_opts, $after_col) = @_;
63 |
64 | # Postgres only allows adding columns at the end, so $after_col is ignored
65 | return "ALTER TABLE $table ADD COLUMN $col_name $col_type $col_opts;"
66 | },
67 |
68 | modify_column => sub {
69 | my ($table, $col_name, $col_type, $col_opts) = @_;
70 | return "ALTER TABLE $table ALTER COLUMN $col_name TYPE $col_type;"
71 | },
72 | );
73 |
74 | 1;
75 |
--------------------------------------------------------------------------------
/dbx_mysql.pl:
--------------------------------------------------------------------------------
1 | %dbx = (
2 | epoch_to_timestamp_fn => 'FROM_UNIXTIME',
3 |
4 | to_hex_string => sub {
5 | my ($bin) = @_;
6 | return "X'" . unpack("H*", $bin) . "'";
7 | },
8 |
9 | column_info_type_col => 'mysql_type_name',
10 |
11 | tables => {
12 | "report" => {
13 | column_definitions => [
14 | "serial" , "int" , "unsigned NOT NULL AUTO_INCREMENT",
15 | "mindate" , "timestamp" , "NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP",
16 | "maxdate" , "timestamp" , "NULL",
17 | "domain" , "varchar(255)" , "NOT NULL",
18 | "org" , "varchar(255)" , "NOT NULL",
19 | "reportid" , "varchar(255)" , "NOT NULL",
20 | "email" , "varchar(255)" , "NULL",
21 | "extra_contact_info" , "varchar(255)" , "NULL",
22 | "policy_adkim" , "varchar(20)" , "NULL",
23 | "policy_aspf" , "varchar(20)" , "NULL",
24 | "policy_p" , "varchar(20)" , "NULL",
25 | "policy_sp" , "varchar(20)" , "NULL",
26 | "policy_pct" , "tinyint" , "unsigned",
27 | "raw_xml" , "mediumtext" , "",
28 | ],
29 | additional_definitions => "PRIMARY KEY (serial), UNIQUE KEY domain (domain, org, reportid)",
30 | table_options => "ROW_FORMAT=COMPRESSED",
31 | indexes => [],
32 | },
33 | "rptrecord" => {
34 | column_definitions => [
35 | "id" , "int" , "unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY",
36 | "serial" , "int" , "unsigned NOT NULL",
37 | "ip" , "int" , "unsigned",
38 | "ip6" , "binary(16)" , "",
39 | "rcount" , "int" , "unsigned NOT NULL",
40 | "disposition" , "enum('" . join("','", ALLOWED_DISPOSITION) . "')" , "",
41 | "reason" , "varchar(255)" , "",
42 | "dkimdomain" , "varchar(255)" , "",
43 | "dkimresult" , "enum('" . join("','", ALLOWED_DKIMRESULT) . "')" , "",
44 | "spfdomain" , "varchar(255)" , "",
45 | "spfresult" , "enum('" . join("','", ALLOWED_SPFRESULT) . "')" , "",
46 | "spf_align" , "enum('" . join("','", ALLOWED_SPF_ALIGN) . "')" , "NOT NULL",
47 | "dkim_align" , "enum('" . join("','", ALLOWED_DKIM_ALIGN) . "')" , "NOT NULL",
48 | "identifier_hfrom" , "varchar(255)" , ""
49 | ],
50 | additional_definitions => "KEY serial (serial, ip), KEY serial6 (serial, ip6)",
51 | table_options => "",
52 | indexes => [],
53 | },
54 | },
55 |
56 | add_column => sub {
57 | my ($table, $col_name, $col_type, $col_opts, $after_col) = @_;
58 |
59 | my $insert_pos = "FIRST";
60 | if ($after_col) {
61 | $insert_pos = "AFTER $after_col";
62 | }
63 | return "ALTER TABLE $table ADD $col_name $col_type $col_opts $insert_pos;"
64 | },
65 |
66 | modify_column => sub {
67 | my ($table, $col_name, $col_type, $col_opts) = @_;
68 | return "ALTER TABLE $table MODIFY COLUMN $col_name $col_type $col_opts;"
69 | },
70 | );
71 |
72 | 1;
73 |
--------------------------------------------------------------------------------
/dmarcts-report-parser.conf.sample:
--------------------------------------------------------------------------------
1 | ################################################################################
2 | ### configuration ##############################################################
3 | ################################################################################
4 |
5 | # If IMAP access is not used, config options starting with $imap do not need to
6 | # be set and are ignored.
7 |
8 | $debug = 0;
9 | $delete_reports = 0;
10 |
11 | # Supported types: mysql, Pg. If unset, defaults to mysql
12 | #$dbtype = 'mysql';
13 | $dbname = 'dmarc';
14 | $dbuser = 'dmarc';
15 | $dbpass = 'password';
16 | $dbhost = 'dbhost'; # Set the hostname if we can't connect to the local socket.
17 | $dbport = '3306';
18 |
19 | $imapserver = 'imap.server';
20 | $imapuser = 'username';
21 | $imappass = 'password';
22 | $imapport = '143';
23 | $imapssl = '0'; # If set to 1, remember to change server port to 993 and disable imaptls.
24 | $imaptls = '1'; # Enabled as the default and best-practice.
25 | $tlsverify = '1'; # Enable verify server cert as the default and best-practice.
26 | $imapignoreerror = '0'; # set it to 1 if you see an "ERROR: message_string()
27 | # expected 119613 bytes but received 81873 you may
28 | # need the IgnoreSizeErrors option" because of malfunction
29 | # imap server as MS Exchange 2007, ...
30 | $imapreadfolder = 'dmarc';
31 |
32 | # If $imapmovefolder is set, processed IMAP messages will be moved (overruled by
33 | # the --delete option!)
34 | $imapmovefolder = 'dmarc/processed';
35 |
36 | # If $imapmovefoldererr is set, IMAP messages that fail will be moved. If unset, failed messages
37 | # will move to $imapmovefolder (if it is set). Overruled by the --delete option!
38 | $imapmovefoldererr = 'Inbox.notProcessed';
39 |
40 | # maximum size of XML files to store in database, long files can cause transaction aborts
41 | $maxsize_xml = 50000;
42 | # store XML as base64 encopded gzip in database (save space, harder usable)
43 | $compress_xml = 0;
44 |
45 | # if there was an error during file processing (message does not contain XML or ZIP parts,
46 | # or a database error) the parser reports an error and does not delete the file, even if
47 | # delete_reports is set (or --delete is given). Deletion can be enforced by delete_failed,
48 | # however not for database errors.
49 | $delete_failed = 0;
50 |
--------------------------------------------------------------------------------
/dmarcts-report-parser.pl:
--------------------------------------------------------------------------------
1 | #!/usr/bin/perl
2 |
3 | ################################################################################
4 | # dmarcts-report-parser - A Perl based tool to parse DMARC reports from an IMAP
5 | # mailbox or from the filesystem, and insert the information into a database.
6 | # ( Formerly known as imap-dmarcts )
7 | #
8 | # Copyright (C) 2016 TechSneeze.com and John Bieling
9 | #
10 | # Available at:
11 | # https://github.com/techsneeze/dmarcts-report-parser
12 | #
13 | # This program is free software: you can redistribute it and/or modify it under
14 | # the terms of the GNU General Public License as published by the Free Software
15 | # Foundation, either version 3 of the License, or (at your option) any later
16 | # version.
17 | #
18 | # This program is distributed in the hope that it will be useful, but WITHOUT
19 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
20 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
21 | #
22 | # You should have received a copy of the GNU General Public License along with
23 | # this program. If not, see .
24 | ################################################################################
25 |
26 | ################################################################################
27 | # The subroutines storeXMLInDatabase() and getXMLFromMessage() are based on
28 | # John R. Levine's rddmarc (http://www.taugh.com/rddmarc/). The following
29 | # special conditions apply to those subroutines:
30 | #
31 | # Copyright 2012, Taughannock Networks. All rights reserved.
32 | #
33 | # Redistribution and use in source and binary forms, with or without
34 | # modification, are permitted provided that the following conditions are met:
35 | #
36 | # Redistributions of source code must retain the above copyright notice, this
37 | # list of conditions and the following disclaimer.
38 | #
39 | # Redistributions in binary form must reproduce the above copyright notice, this
40 | # list of conditions and the following disclaimer in the documentation and/or
41 | # other materials provided with the distribution.
42 | #
43 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
44 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
46 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
47 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
49 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
50 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
51 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
52 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
53 | ################################################################################
54 |
55 | # Always be safe
56 | use strict;
57 | use warnings;
58 |
59 | # Use these modules
60 | use Getopt::Long;
61 | use IO::Compress::Gzip qw(gzip $GzipError);
62 | #use Data::Dumper;
63 | use Mail::IMAPClient;
64 | use Mail::Mbox::MessageParser;
65 | use MIME::Base64 qw(encode_base64);
66 | use MIME::Words qw(decode_mimewords);
67 | use MIME::Parser;
68 | use MIME::Parser::Filer;
69 | use XML::Simple;
70 | use DBI;
71 | use Socket;
72 | use Socket6;
73 | use PerlIO::gzip;
74 | use File::Basename ();
75 | use File::MimeInfo;
76 | use IO::Socket::SSL;
77 | #use IO::Socket::SSL 'debug3';
78 |
79 |
80 |
81 | ################################################################################
82 | ### usage ######################################################################
83 | ################################################################################
84 |
85 | sub show_usage {
86 | print "\n";
87 | print " Usage: \n";
88 | print " ./dmarcts-report-parser.pl [OPTIONS] [PATH] \n";
89 | print "\n";
90 | print " This script needs a configuration file called in \n";
91 | print " the current working directory, which defines a database server with credentials \n";
92 | print " and (if used) an IMAP server with credentials. \n";
93 | print "\n";
94 | print " Additionaly, one of the following source options must be provided: \n";
95 | print " -i : Read reports from messages on IMAP server as defined in the \n";
96 | print " config file. \n";
97 | print " -m : Read reports from mbox file(s) provided in PATH. \n";
98 | print " -e : Read reports from MIME email file(s) provided in PATH. \n";
99 | print " -x : Read reports from xml file(s) provided in PATH. \n";
100 | print " -z : Read reports from zip file(s) provided in PATH. \n";
101 | print "\n";
102 | print " The following optional options are allowed: \n";
103 | print " -d : Print debug info. \n";
104 | print " -r : Replace existing reports rather than skipping them. \n";
105 | print " --delete : Delete processed message files (the XML is stored in the \n";
106 | print " database for later reference). \n";
107 | print " --info : Print out number of XML files or emails processed. \n";
108 | print "\n";
109 | }
110 |
111 |
112 |
113 |
114 |
115 | ################################################################################
116 | ### main #######################################################################
117 | ################################################################################
118 |
119 | # Define all possible configuration options.
120 | our ($debug, $delete_reports, $delete_failed, $reports_replace, $maxsize_xml, $compress_xml,
121 | $dbtype, $dbname, $dbuser, $dbpass, $dbhost, $dbport, $db_tx_support,
122 | $imapserver, $imapport, $imapuser, $imappass, $imapignoreerror, $imapssl, $imaptls, $imapmovefolder,
123 | $imapmovefoldererr, $imapreadfolder, $imapopt, $tlsverify, $processInfo);
124 |
125 | # defaults
126 | $maxsize_xml = 50000;
127 | $dbtype = 'mysql';
128 | $db_tx_support = 1;
129 |
130 | # used in messages
131 | my $scriptname = 'dmarcts-report-parser.pl';
132 |
133 | # allowed values for the DB columns, also used to build the enum() in the
134 | # CREATE TABLE statements in checkDatabase(), in order defined here
135 | use constant ALLOWED_DISPOSITION => qw(
136 | none
137 | quarantine
138 | reject
139 | unknown
140 | );
141 | use constant ALLOWED_DKIM_ALIGN => qw(
142 | fail
143 | pass
144 | unknown
145 | );
146 | use constant ALLOWED_SPF_ALIGN => qw(
147 | fail
148 | pass
149 | unknown
150 | );
151 | use constant ALLOWED_DKIMRESULT => qw(
152 | none
153 | pass
154 | fail
155 | neutral
156 | policy
157 | temperror
158 | permerror
159 | unknown
160 | );
161 | use constant ALLOWED_SPFRESULT => qw(
162 | none
163 | neutral
164 | pass
165 | fail
166 | softfail
167 | temperror
168 | permerror
169 | unknown
170 | );
171 |
172 | # Load script configuration options from local config file. The file is expected
173 | # to be in the current working directory.
174 | my $conf_file = 'dmarcts-report-parser.conf';
175 |
176 | # Get command line options.
177 | my %options = ();
178 | use constant { TS_IMAP => 0, TS_MESSAGE_FILE => 1, TS_XML_FILE => 2, TS_MBOX_FILE => 3, TS_ZIP_FILE => 4 };
179 | GetOptions( \%options, 'd', 'r', 'x', 'm', 'e', 'i', 'z', 'delete', 'info', 'c' => \$conf_file );
180 |
181 | # locate conf file or die
182 | if ( -e $conf_file ) {
183 | #$conf_file = "./$conf_file";
184 | } elsif( -e (File::Basename::dirname($0) . "/$conf_file" ) ) {
185 | $conf_file = ( File::Basename::dirname($0) . "/$conf_file" );
186 | } else {
187 | show_usage();
188 | die "$scriptname: Could not read config file '$conf_file' from current working directory or path (" . File::Basename::dirname($0) . ')'
189 | }
190 |
191 | # load conf file with error handling
192 | if ( substr($conf_file, 0, 1) ne '/' and substr($conf_file, 0, 1) ne '.') {
193 | $conf_file = "./$conf_file";
194 | }
195 | my $conf_return = do $conf_file;
196 | die "$scriptname: couldn't parse $conf_file: $@" if $@;
197 | die "$scriptname: couldn't do $conf_file: $!" unless defined $conf_return;
198 |
199 | # check config
200 | if (!defined $imapreadfolder ) {
201 | die "$scriptname: \$imapreadfolder not defined. Check config file";
202 | }
203 | if (!defined $imapignoreerror ) {
204 | $imapignoreerror = 0; # maintain compatibility to old version
205 | }
206 |
207 | # Evaluate command line options
208 | my $source_options = 0;
209 | our $reports_source;
210 |
211 | if (exists $options{m}) {
212 | $source_options++;
213 | $reports_source = TS_MBOX_FILE;
214 | }
215 |
216 | if (exists $options{x}) {
217 | $source_options++;
218 | $reports_source = TS_XML_FILE;
219 | }
220 |
221 | if (exists $options{e}) {
222 | $source_options++;
223 | $reports_source = TS_MESSAGE_FILE;
224 | }
225 |
226 | if (exists $options{i}) {
227 | $source_options++;
228 | $reports_source = TS_IMAP;
229 | }
230 |
231 | if (exists $options{z}) {
232 | $source_options++;
233 | $reports_source = TS_ZIP_FILE;
234 | }
235 |
236 | if (exists $options{c}) {
237 | $source_options++;
238 | }
239 |
240 | if ($source_options > 1) {
241 | show_usage();
242 | die "$scriptname: Only one source option can be used (-i, -x, -m, -e or -z).\n";
243 | } elsif ($source_options == 0) {
244 | show_usage();
245 | die "$scriptname: Please provide a source option (-i, -x, -m, -e or -z).\n";
246 | }
247 |
248 | if ($ARGV[0]) {
249 | if ($reports_source == TS_IMAP) {
250 | show_usage();
251 | die "$scriptname: The IMAP source option (-i) may not be used together with a PATH.\n";
252 | }
253 | } else {
254 | if ($reports_source != TS_IMAP && $source_options == 1) {
255 | show_usage();
256 | die "$scriptname: The provided source option requires a PATH.\n";
257 | }
258 | }
259 |
260 | # Override config options by command line options.
261 | if (exists $options{r}) {$reports_replace = 1;}
262 | if (exists $options{d}) {$debug = 1;}
263 | if (exists $options{delete}) {$delete_reports = 1;}
264 | if (exists $options{info}) {$processInfo = 1;}
265 |
266 | # Setup connection to database server.
267 | our %dbx;
268 | my $dbx_file = File::Basename::dirname($0) . "/dbx_$dbtype.pl";
269 | my $dbx_return = do $dbx_file;
270 | die "$scriptname: couldn't load DB definition for type $dbtype: $@" if $@;
271 | die "$scriptname: couldn't load DB definition for type $dbtype: $!" unless defined $dbx_return;
272 |
273 | my $dbh = DBI->connect("DBI:$dbtype:database=$dbname;host=$dbhost;port=$dbport",
274 | $dbuser, $dbpass)
275 | or die "$scriptname: Cannot connect to database\n";
276 | if ($db_tx_support) {
277 | $dbh->{AutoCommit} = 0;
278 | }
279 | checkDatabase($dbh);
280 |
281 |
282 | # Process messages based on $reports_source.
283 | if ($reports_source == TS_IMAP) {
284 | my $socketargs = '';
285 | my $processedReport = 0;
286 |
287 | # Disable verify mode for TLS support.
288 | if ($imaptls == 1) {
289 | if ( $tlsverify == 0 ) {
290 | print "use tls without verify servercert.\n" if $debug;
291 | $imapopt = [ SSL_verify_mode => SSL_VERIFY_NONE ];
292 | } else {
293 | print "use tls with verify servercert.\n" if $debug;
294 | $imapopt = [ SSL_verify_mode => SSL_VERIFY_PEER ];
295 | }
296 | # The whole point of setting this socket arg is so that we don't get the nasty warning
297 | } else {
298 | print "using ssl without verify servercert.\n" if $debug;
299 | $socketargs = [ SSL_verify_mode => SSL_VERIFY_NONE ];
300 | }
301 |
302 | print "connection to $imapserver with Ssl => $imapssl, User => $imapuser, Ignoresizeerrors => $imapignoreerror\n" if $debug;
303 |
304 | # Setup connection to IMAP server.
305 | my $imap = Mail::IMAPClient->new(
306 | Server => $imapserver,
307 | Port => $imapport,
308 | Ssl => $imapssl,
309 | Starttls => $imapopt,
310 | Debug => $debug,
311 | Socketargs => $socketargs
312 | )
313 | # module uses eval, so we use $@ instead of $!
314 | or die "$scriptname: IMAP Failure: $@";
315 |
316 | # This connection is finished this way because of the tradgedy of exchange...
317 | $imap->User($imapuser);
318 | $imap->Password($imappass);
319 | $imap->connect() or die "$scriptname: Could not connect: $@";
320 |
321 | # Ignore Size Errors if we're using Exchange
322 | $imap->Ignoresizeerrors($imapignoreerror);
323 |
324 | # Set $imap to UID mode, which will force imap functions to use/return
325 | # UIDs, instead of message sequence numbers. UIDs are not allowed to
326 | # change during a session and are not allowed to be used twice. Looping
327 | # over message sequence numbers and deleting a msg in between could have
328 | # unwanted side effects.
329 | $imap->Uid(1);
330 |
331 | # How many msgs are we going to process?
332 | print "Processing ". $imap->message_count($imapreadfolder)." messages in folder <$imapreadfolder>.\n" if $debug;
333 |
334 | # Only select and search $imapreadfolder, if we actually
335 | # have something to do.
336 | if ($imap->message_count($imapreadfolder)) {
337 | # Select the mailbox to get messages from.
338 | $imap->select($imapreadfolder)
339 | or die "$scriptname: IMAP Select Error: $!";
340 |
341 | # Store each message as an array element.
342 | my @msgs = $imap->search('ALL')
343 | or die "$scriptname: Couldn't get all messages\n";
344 |
345 | # Loop through IMAP messages.
346 | foreach my $msg (@msgs) {
347 |
348 | my $processResult = processXML(TS_MESSAGE_FILE, $imap->message_string($msg), "IMAP message with UID #".$msg);
349 | $processedReport++;
350 | if ($processResult & 4) {
351 | # processXML returned a value with database error bit enabled, do nothing at all!
352 | if ($imapmovefoldererr) {
353 | # if we can, move to error folder
354 | moveToImapFolder($imap, $msg, $imapmovefoldererr);
355 | } else {
356 | # do nothing at all
357 | next;
358 | }
359 | } elsif ($processResult & 2) {
360 | # processXML return a value with delete bit enabled.
361 | $imap->delete_message($msg)
362 | or warn "$scriptname: Could not delete IMAP message. [$@]\n";
363 | } elsif ($imapmovefolder) {
364 | if ($processResult & 1 || !$imapmovefoldererr) {
365 | # processXML processed the XML OK, or it failed and there is no error imap folder
366 | moveToImapFolder($imap, $msg, $imapmovefolder);
367 | } elsif ($imapmovefoldererr) {
368 | # processXML failed and error folder set
369 | moveToImapFolder($imap, $msg, $imapmovefoldererr);
370 | }
371 | } elsif ($imapmovefoldererr && !($processResult & 1)) {
372 | # processXML failed, error imap folder set, but imapmovefolder unset. An unlikely setup, but still...
373 | moveToImapFolder($imap, $msg, $imapmovefoldererr);
374 | }
375 | }
376 |
377 | # Expunge and close the folder.
378 | $imap->expunge($imapreadfolder);
379 | $imap->close($imapreadfolder);
380 | }
381 |
382 | # We're all done with IMAP here.
383 | $imap->logout();
384 | if ( $debug || $processInfo ) { print "$scriptname: Processed $processedReport emails.\n"; }
385 |
386 | } else { # TS_MESSAGE_FILE or TS_XML_FILE or TS_MBOX_FILE
387 |
388 | my $counts = 0;
389 | foreach my $a (@ARGV) {
390 | # Linux bash supports wildcard expansion BEFORE the script is
391 | # called, so here we only see a list of files. Other OS behave
392 | # different, so we should not depend on that feature: Use glob
393 | # on each argument to manually expand the argument, if possible.
394 | my @file_list = glob($a);
395 |
396 | foreach my $f (@file_list) {
397 | my $filecontent;
398 |
399 | if ($reports_source == TS_MBOX_FILE) {
400 | my $parser = Mail::Mbox::MessageParser->new({"file_name" => $f, "debug" => $debug, "enable_cache" => 0});
401 | my $num = 0;
402 |
403 | do {
404 | $num++;
405 | $filecontent = $parser->read_next_email();
406 | if (defined($filecontent)) {
407 | if (processXML(TS_MESSAGE_FILE, $filecontent, "message #$num of mbox file <$f>") & 2) {
408 | # processXML return a value with delete bit enabled
409 | warn "$scriptname: Removing message #$num from mbox file <$f> is not yet supported.\n";
410 | }
411 | $counts++;
412 | }
413 | } while(defined($filecontent));
414 |
415 | } elsif ($reports_source == TS_ZIP_FILE) {
416 | # filecontent is zip file
417 | $filecontent = getXMLFromZip($f);
418 | if (processXML(TS_ZIP_FILE, $filecontent, "xml file <$f>") & 2) {
419 | # processXML return a value with delete bit enabled
420 | unlink($f);
421 | }
422 | $counts++;
423 | } elsif (open(FILE, "<", $f)) {
424 |
425 | $filecontent = join("", );
426 | close FILE;
427 |
428 | if ($reports_source == TS_MESSAGE_FILE) {
429 | # filecontent is a mime message with zip or xml part
430 | if (processXML(TS_MESSAGE_FILE, $filecontent, "message file <$f>") & 2) {
431 | # processXML return a value with delete bit enabled
432 | unlink($f);
433 | }
434 | $counts++;
435 | } elsif ($reports_source == TS_XML_FILE) {
436 | # filecontent is xml file
437 | if (processXML(TS_XML_FILE, $filecontent, "xml file <$f>") & 2) {
438 | # processXML return a value with delete bit enabled
439 | unlink($f);
440 | }
441 | $counts++;
442 | } else {
443 | warn "$scriptname: Unknown reports_source <$reports_source> for file <$f>. Skipped.\n";
444 | }
445 |
446 | } else {
447 | warn "$scriptname: Could not open file <$f>: $!. Skipped.\n";
448 | # Could not retrieve filecontent, the skipped message
449 | # will be processed every time the script is run even if
450 | # delete_reports and delete_failed is given. The user
451 | # has to look at the actual file.
452 | }
453 | }
454 | }
455 | if ($debug || $processInfo) { print "$scriptname: Processed $counts messages(s).\n"; }
456 | }
457 |
458 |
459 |
460 | ################################################################################
461 | ### subroutines ################################################################
462 | ################################################################################
463 |
464 | sub moveToImapFolder {
465 | my $imap = $_[0];
466 | my $msg = $_[1];
467 | my $imapfolder = $_[2];
468 |
469 | print "Moving (copy and delete) IMAP message file to IMAP folder: $imapfolder\n" if $debug;
470 |
471 | # Try to create $imapfolder, if it does not exist.
472 | if (!$imap->exists($imapfolder)) {
473 | $imap->create($imapfolder)
474 | or warn "$scriptname: Could not create IMAP folder: $imapfolder.\n";
475 | }
476 |
477 | # Try to move the message to $imapfolder.
478 | my $newid = $imap->copy($imapfolder, [ $msg ]);
479 | if (!$newid) {
480 | warn "$scriptname: Error on moving (copy and delete) processed IMAP message: Could not COPY message to IMAP folder: <$imapfolder>!\n";
481 | warn "$scriptname: Messsage will not be moved/deleted. [$@]\n";
482 | } else {
483 | $imap->delete_message($msg)
484 | or do {
485 | warn "$scriptname: Error on moving (copy and delete) processed IMAP message: Could not DELETE message\n";
486 | warn "$scriptname: after copying it to <$imapfolder>. [$@]\n";
487 | }
488 | }
489 | }
490 |
491 | sub processXML {
492 | my ($type, $filecontent, $f) = (@_);
493 |
494 | if ($debug) {
495 | print "\n";
496 | print "----------------------------------------------------------------\n";
497 | print "Processing $f \n";
498 | print "----------------------------------------------------------------\n";
499 | print "Type: $type\n";
500 | print "FileContent: $filecontent\n";
501 | print "MSG: $f\n";
502 | print "----------------------------------------------------------------\n";
503 | }
504 |
505 | my $xml; #TS_XML_FILE or TS_MESSAGE_FILE
506 | if ($type == TS_MESSAGE_FILE) {$xml = getXMLFromMessage($filecontent);}
507 | elsif ($type == TS_ZIP_FILE) {$xml = $filecontent;}
508 | else {$xml = getXMLFromXMLString($filecontent);}
509 |
510 | # If !$xml, the file/mail is probably not a DMARC report.
511 | # So do not storeXMLInDatabase.
512 | if ($xml && storeXMLInDatabase($xml) <= 0) {
513 | # If storeXMLInDatabase returns false, there was some sort
514 | # of database storage failure and we MUST NOT delete the
515 | # file, because it has not been pushed into the database.
516 | # The user must investigate this issue.
517 | warn "$scriptname: Skipping $f due to database errors.\n";
518 | return 5; #xml ok(1), but database error(4), thus no delete (!2)
519 | }
520 |
521 | # Delete processed message, if the --delete option
522 | # is given. Failed reports are only deleted, if delete_failed is given.
523 | if ($delete_reports && ($xml || $delete_failed)) {
524 | if ($xml) {
525 | print "Removing after report has been processed.\n" if $debug;
526 | return 3; #xml ok (1), delete file (2)
527 | } else {
528 | # A mail which does not look like a DMARC report
529 | # has been processed and should now be deleted.
530 | # Print its content so it gets send as cron
531 | # message, so the user can still investigate.
532 | warn "$scriptname: The $f does not seem to contain a valid DMARC report. Skipped and Removed. Content:\n";
533 | warn $filecontent."\n";
534 | return 2; #xml not ok (!1), delete file (2)
535 | }
536 | }
537 |
538 | if ($xml) {
539 | return 1;
540 | } else {
541 | warn "$scriptname: The $f does not seem to contain a valid DMARC report. Skipped.\n";
542 | return 0;
543 | }
544 | }
545 |
546 |
547 | ################################################################################
548 |
549 | # Walk through a mime message and return a reference to the XML data containing
550 | # the fields of the first ZIPed XML file embedded into the message. The XML
551 | # itself is not checked to be a valid DMARC report.
552 | sub getXMLFromMessage {
553 | my ($message) = (@_);
554 |
555 | # fixup type in trustwave SEG mails
556 | $message =~ s/ContentType:/Content-Type:/;
557 |
558 | my $parser = new MIME::Parser;
559 | $parser->output_dir("/tmp");
560 | $parser->filer->ignore_filename(1);
561 | my $ent = $parser->parse_data($message);
562 |
563 | my $body = $ent->bodyhandle;
564 | my $mtype = $ent->mime_type;
565 | my $subj = decode_mimewords($ent->get('subject'));
566 | chomp($subj); # Subject always contains a \n.
567 |
568 | if ($debug) {
569 | print "Subject: $subj\n";
570 | print "MimeType: $mtype\n";
571 | }
572 |
573 | my $location;
574 | my $isgzip = 0;
575 |
576 | if(lc $mtype eq "application/zip") {
577 | if ($debug) {
578 | print "This is a ZIP file \n";
579 | }
580 |
581 | $location = $body->path;
582 |
583 | } elsif (lc $mtype eq "application/gzip" or lc $mtype eq "application/x-gzip") {
584 | if ($debug) {
585 | print "This is a GZIP file \n";
586 | }
587 |
588 | $location = $body->path;
589 | $isgzip = 1;
590 |
591 | } elsif (lc $mtype =~ "multipart/") {
592 | # At the moment, nease.net messages are multi-part, so we need
593 | # to breakdown the attachments and find the zip.
594 | if ($debug) {
595 | print "This is a multipart attachment \n";
596 | }
597 | #print Dumper($ent->parts);
598 |
599 | my $num_parts = $ent->parts;
600 | for (my $i=0; $i < $num_parts; $i++) {
601 | my $part = $ent->parts($i);
602 |
603 | # Find a zip file to work on...
604 | if(lc $part->mime_type eq "application/gzip" or lc $part->mime_type eq "application/x-gzip") {
605 | $location = $ent->parts($i)->{ME_Bodyhandle}->{MB_Path};
606 | $isgzip = 1;
607 | print "$location\n" if $debug;
608 | last; # of parts
609 | } elsif(lc $part->mime_type eq "application/x-zip-compressed"
610 | or $part->mime_type eq "application/zip") {
611 |
612 | $location = $ent->parts($i)->{ME_Bodyhandle}->{MB_Path};
613 | print "$location\n" if $debug;
614 | } elsif(lc $part->mime_type eq "application/octet-stream") {
615 | $location = $ent->parts($i)->{ME_Bodyhandle}->{MB_Path};
616 | $isgzip = 1 if $location =~ /\.gz$/;
617 | print "$location\n" if $debug;
618 | } else {
619 | # Skip the attachment otherwise.
620 | if ($debug) {
621 | print "Skipped an unknown attachment (".lc $part->mime_type.")\n";
622 | }
623 | next; # of parts
624 | }
625 | }
626 | } else {
627 | ## Clean up dangling mime parts in /tmp of messages without ZIP.
628 | my $num_parts = $ent->parts;
629 | for (my $i=0; $i < $num_parts; $i++) {
630 | if ($debug) {
631 | if ($ent->parts($i)->{ME_Bodyhandle} && $ent->parts($i)->{ME_Bodyhandle}->{MB_Path}) {
632 | print $ent->parts($i)->{ME_Bodyhandle}->{MB_Path};
633 | } else {
634 | print "undef";
635 | }
636 | print "\n";
637 | }
638 | if($ent->parts($i)->{ME_Bodyhandle}) {$ent->parts($i)->{ME_Bodyhandle}->purge;}
639 | }
640 | }
641 |
642 |
643 | # If a ZIP has been found, extract XML and parse it.
644 | my $xml;
645 | if(defined($location)) {
646 | if ($debug) {
647 | print "body is in " . $location . "\n";
648 | }
649 |
650 | # Open the zip file and process the XML contained inside.
651 | my $unzip = "";
652 | if($isgzip) {
653 | open(XML, "<:gzip", $location)
654 | or $unzip = "ungzip";
655 | } else {
656 | open(XML, "-|", "unzip", "-p", $location)
657 | or $unzip = "unzip"; # Will never happen.
658 |
659 | # Sadly unzip -p never failes, but we can check if the
660 | # filehandle points to an empty file and pretend it did
661 | # not open/failed.
662 | if (eof XML) {
663 | $unzip = "unzip";
664 | }
665 | }
666 |
667 | # Read XML if possible (if open)
668 | if ($unzip eq "") {
669 | $xml = getXMLFromXMLString(join("", ));
670 | if (!$xml) {
671 | warn "$scriptname: Subject: $subj\n:";
672 | warn "$scriptname: The XML found in ZIP file (temp. location: <$location>) does not seem to be valid XML! \n";
673 | }
674 | close XML;
675 | } else {
676 | warn "$scriptname: Subject: $subj\n:";
677 | warn "$scriptname: Failed to $unzip ZIP file (temp. location: <$location>)! \n";
678 | close XML;
679 | }
680 | } else {
681 | warn "$scriptname: Subject: $subj\n:";
682 | warn "$scriptname: Could not find an embedded ZIP! \n";
683 | }
684 |
685 | if($body) {$body->purge;}
686 | if($ent) {$ent->purge;}
687 | return $xml;
688 | }
689 |
690 | ################################################################################
691 |
692 | sub getXMLFromZip {
693 | my $filename = $_[0];
694 | my $mtype = mimetype($filename);
695 |
696 | if ($debug) {
697 | print "Filename: $filename, MimeType: $mtype\n";
698 | }
699 |
700 | my $isgzip = 0;
701 |
702 | if(lc $mtype eq "application/zip") {
703 | if ($debug) {
704 | print "This is a ZIP file \n";
705 | }
706 | } elsif (lc $mtype eq "application/gzip" or lc $mtype eq "application/x-gzip") {
707 | if ($debug) {
708 | print "This is a GZIP file \n";
709 | }
710 |
711 | $isgzip = 1;
712 | } else {
713 | if ($debug) {
714 | print "This is not an archive file \n";
715 | }
716 | }
717 |
718 | # If a ZIP has been found, extract XML and parse it.
719 | my $xml;
720 | if(defined($filename)) {
721 | # Open the zip file and process the XML contained inside.
722 | my $unzip = "";
723 | if($isgzip) {
724 | open(XML, "<:gzip", $filename)
725 | or $unzip = "ungzip";
726 | } else {
727 | open(XML, "-|", "unzip", "-p", $filename)
728 | or $unzip = "unzip"; # Will never happen.
729 |
730 | # Sadly unzip -p never failes, but we can check if the
731 | # filehandle points to an empty file and pretend it did
732 | # not open/failed.
733 | if (eof XML) {
734 | $unzip = "unzip";
735 | }
736 | }
737 |
738 | # Read XML if possible (if open)
739 | if ($unzip eq "") {
740 | $xml = getXMLFromXMLString(join("", ));
741 | if (!$xml) {
742 | warn "$scriptname: The XML found in ZIP file (<$filename>) does not seem to be valid XML! \n";
743 | }
744 | close XML;
745 | } else {
746 | warn "$scriptname: Failed to $unzip ZIP file (<$filename>)! \n";
747 | close XML;
748 | }
749 | } else {
750 | warn "$scriptname: Could not find an <$filename>! \n";
751 | }
752 |
753 | return $xml;
754 | }
755 |
756 | ################################################################################
757 |
758 | sub getXMLFromXMLString {
759 | my $raw_xml = $_[0];
760 |
761 | eval {
762 | my $xs = XML::Simple->new();
763 | my $ref = $xs->XMLin($raw_xml, SuppressEmpty => '');
764 | $ref->{'raw_xml'} = $raw_xml;
765 |
766 | return $ref;
767 | } or do {
768 | return undef;
769 | }
770 | }
771 |
772 |
773 | ################################################################################
774 |
775 | # Extract fields from the XML report data hash and store them into the database.
776 | # return 1 when ok, 0, for serious error and -1 for minor errors
777 | sub storeXMLInDatabase {
778 | my $xml = $_[0]; # $xml is a reference to the xml data
779 |
780 | my $from = $xml->{'report_metadata'}->{'date_range'}->{'begin'};
781 | my $to = $xml->{'report_metadata'}->{'date_range'}->{'end'};
782 | my $org = $xml->{'report_metadata'}->{'org_name'};
783 | my $id = $xml->{'report_metadata'}->{'report_id'};
784 | my $email = $xml->{'report_metadata'}->{'email'};
785 | my $extra = $xml->{'report_metadata'}->{'extra_contact_info'};
786 | my $domain = undef;
787 | my $policy_adkim = undef;
788 | my $policy_aspf = undef;
789 | my $policy_p = undef;
790 | my $policy_sp = undef;
791 | my $policy_pct = undef;
792 |
793 | if (ref $xml->{'policy_published'} eq "HASH") {
794 | $domain = $xml->{'policy_published'}->{'domain'};
795 | $policy_adkim = $xml->{'policy_published'}->{'adkim'};
796 | $policy_aspf = $xml->{'policy_published'}->{'aspf'};
797 | $policy_p = $xml->{'policy_published'}->{'p'};
798 | $policy_sp = $xml->{'policy_published'}->{'sp'};
799 | $policy_pct = $xml->{'policy_published'}->{'pct'};
800 | } else {
801 | $domain = $xml->{'policy_published'}[0]->{'domain'};
802 | $policy_adkim = $xml->{'policy_published'}[0]->{'adkim'};
803 | $policy_aspf = $xml->{'policy_published'}[0]->{'aspf'};
804 | $policy_p = $xml->{'policy_published'}[0]->{'p'};
805 | $policy_sp = $xml->{'policy_published'}[0]->{'sp'};
806 | $policy_pct = $xml->{'policy_published'}[0]->{'pct'};
807 | }
808 |
809 | my $record = $xml->{'record'};
810 | if ( ! defined($record) ) {
811 | warn "$scriptname: $org: $id: No records in report. Skipped.\n";
812 | return 0;
813 | }
814 |
815 | # see if already stored
816 | my $sth = $dbh->prepare(qq{SELECT org, serial FROM report WHERE reportid=? AND org=? AND domain=?});
817 | $sth->execute($id, $org, $domain);
818 | while ( my ($xorg,$sid) = $sth->fetchrow_array() )
819 | {
820 | if ($reports_replace) {
821 | # $sid is the serial of a report with reportid=$id
822 | # Remove this $sid from rptrecord and report table, but
823 | # try to continue on failure rather than skipping.
824 | print "$scriptname: $org: $id: Replacing data.\n";
825 | $dbh->do(qq{DELETE from rptrecord WHERE serial=?}, undef, $sid);
826 | if ($dbh->errstr) {
827 | warn "$scriptname: $org: $id: Cannot remove report data from database. Try to continue.\n";
828 | }
829 | $dbh->do(qq{DELETE from report WHERE serial=?}, undef, $sid);
830 | if ($dbh->errstr) {
831 | warn "$scriptname: $org: $id: Cannot remove report from database. Try to continue.\n";
832 | }
833 | } else {
834 | print "$scriptname: $org: $id: Already have report, skipped\n";
835 | # Do not store in DB, but return true, so the message can
836 | # be moved out of the way, if configured to do so.
837 | return 1;
838 | }
839 | }
840 |
841 | my $sql = qq{INSERT INTO report(mindate,maxdate,domain,org,reportid,email,extra_contact_info,policy_adkim, policy_aspf, policy_p, policy_sp, policy_pct, raw_xml)
842 | VALUES($dbx{epoch_to_timestamp_fn}(?),$dbx{epoch_to_timestamp_fn}(?),?,?,?,?,?,?,?,?,?,?,?)};
843 | my $storexml = $xml->{'raw_xml'};
844 | if ($compress_xml) {
845 | my $gzipdata;
846 | if(!gzip(\$storexml => \$gzipdata)) {
847 | warn "$scriptname: $org: $id: Cannot add gzip XML to database ($GzipError). Skipped.\n";
848 | rollback($dbh);
849 | return 0;
850 | $storexml = "";
851 | } else {
852 | $storexml = encode_base64($gzipdata, "");
853 | }
854 | }
855 | if (length($storexml) > $maxsize_xml) {
856 | warn "$scriptname: $org: $id: Skipping storage of large XML (".length($storexml)." bytes) as defined in config file.\n";
857 | $storexml = "";
858 | }
859 | $dbh->do($sql, undef, $from, $to, $domain, $org, $id, $email, $extra, $policy_adkim, $policy_aspf, $policy_p, $policy_sp, $policy_pct, $storexml);
860 | if ($dbh->errstr) {
861 | warn "$scriptname: $org: $id: Cannot add report to database. Skipped.\n";
862 | rollback($dbh);
863 | return 0;
864 | }
865 |
866 | my $serial = $dbh->last_insert_id(undef, undef, 'report', undef);
867 | if ($debug){
868 | print " serial $serial \n";
869 | }
870 | sub dorow($$$$) {
871 | my ($serial,$recp,$org,$id) = @_;
872 | my %r = %$recp;
873 |
874 | my $ip = $r{'row'}->{'source_ip'};
875 | if ( $ip eq '' ) {
876 | warn "$scriptname: $org: $id: source_ip is empty. Skipped.\n";
877 | rollback($dbh);
878 | return 0;
879 | }
880 | my $count = $r{'row'}->{'count'};
881 | my $disp = $r{'row'}->{'policy_evaluated'}->{'disposition'};
882 | if ( ! grep { $_ eq $disp } ALLOWED_DISPOSITION ) {
883 | $disp = 'unknown';
884 | };
885 | # some reports don't have dkim/spf, "unknown" is default for these
886 | my $dkim_align = $r{'row'}->{'policy_evaluated'}->{'dkim'};
887 | if ( ! grep { $_ eq $dkim_align } ALLOWED_DKIM_ALIGN ) {
888 | $dkim_align = 'unknown';
889 | };
890 | my $spf_align = $r{'row'}->{'policy_evaluated'}->{'spf'};
891 | if ( ! grep { $_ eq $spf_align } ALLOWED_SPF_ALIGN ) {
892 | $spf_align = 'unknown';
893 | };
894 |
895 | my $identifier_hfrom = $r{'identifiers'}->{'header_from'};
896 |
897 | my ($dkim, $dkimresult, $spf, $spfresult, $reason);
898 | if(ref $r{'auth_results'} ne "HASH"){
899 | warn "$scriptname: $org: $id: Report has no auth_results data. Skipped.\n";
900 | rollback($dbh);
901 | return 0;
902 | }
903 | my $rp = $r{'auth_results'}->{'dkim'};
904 | if(ref $rp eq "HASH") {
905 | $dkim = $rp->{'domain'};
906 | $dkim = undef if ref $dkim eq "HASH";
907 | $dkimresult = $rp->{'result'};
908 | } else { # array, i.e. multiple dkim results (usually from multiple domains)
909 | # glom sigs together
910 | $dkim = join '/',map { my $d = $_->{'domain'}; ref $d eq "HASH"?"": $d } @$rp;
911 | # report results
912 | my $rp_len = scalar(@$rp);
913 | for ( my $i=0; $i < $rp_len; $i++ ) {
914 | if ( $rp->[$i]->{'result'} eq "pass" ) {
915 | # If any one dkim result is a "pass", this should yield an overall "pass" and immediately exit the for loop, ignoring any remaing results
916 | # See
917 | # RFC 6376, DomainKeys Identified Mail (DKIM) Signatures
918 | # Section 4.2: https://tools.ietf.org/html/rfc6376#section-4.2 and
919 | # Section 6.1: https://tools.ietf.org/html/rfc6376#section-6.1
920 | # And the GitHub issues at
921 | # https://github.com/techsneeze/dmarcts-report-viewer/issues/47
922 | # https://github.com/techsneeze/dmarcts-report-parser/pull/78
923 | $dkimresult = "pass";
924 | last;
925 | } else {
926 | for ( my $j=$i+1; $j < $rp_len; $j++ ) {
927 | if ( $rp->[$i]->{'result'} eq $rp->[$j]->{'result'} ) {
928 | # Compare each dkim result to the next one to see if all of the dkim results are the same.
929 | # If all of the dkim results are the same, that will be the overall result.
930 | # If any of them are different, and don't contain a "pass" result, then $dkimresult will be empty
931 | $dkimresult = $rp->[0]->{'result'};
932 | } else {
933 | $dkimresult = 'unknown';
934 | }
935 | }
936 | }
937 | }
938 | }
939 | if ( ! defined($dkimresult) || ! grep { $_ eq $dkimresult } ALLOWED_DKIMRESULT ) {
940 | $dkimresult = 'unknown';
941 | };
942 |
943 | $rp = $r{'auth_results'}->{'spf'};
944 | if(ref $rp eq "HASH") {
945 | $spf = $rp->{'domain'};
946 | $spf = undef if ref $spf eq "HASH";
947 | $spfresult = $rp->{'result'};
948 | } else { # array, i.e. multiple dkim results (usually from multiple domains)
949 | # glom sigs together
950 | $spf = join '/',map { my $d = $_->{'domain'}; ref $d eq "HASH"?"": $d } @$rp;
951 | # report results
952 | my $rp_len = scalar(@$rp);
953 | for ( my $i=0; $i < $rp_len; $i++ ) {
954 | if ( $rp->[$i]->{'result'} eq "pass" ) {
955 | # If any one spf result is a "pass", this should yield an overall "pass" and immediately exit the for loop, ignoring any remaing results
956 | $spfresult = "pass";
957 | last;
958 | } else {
959 | for ( my $j=$i+1; $j < $rp_len; $j++ ) {
960 | if ( $rp->[$i]->{'result'} eq $rp->[$j]->{'result'} ) {
961 | # Compare each spf result to the next one to see if all of the spf results are the same.
962 | # If all of the spf results are the same, that will be the overall result.
963 | # If any of them are different, and don't contain a "pass" result, then $spfresult will be empty
964 | $spfresult = $rp->[0]->{'result'};
965 | } else {
966 | $spfresult = 'unknown';
967 | }
968 | }
969 | }
970 | }
971 | }
972 | if ( ! defined($spfresult) || ! grep { $_ eq $spfresult } ALLOWED_SPFRESULT ) {
973 | $spfresult = 'unknown';
974 | };
975 |
976 | $rp = $r{'row'}->{'policy_evaluated'}->{'reason'};
977 | if(ref $rp eq "HASH") {
978 | $reason = $rp->{'type'};
979 | } else {
980 | $reason = join '/',map { $_->{'type'} } @$rp;
981 | }
982 | #print "ip=$ip, count=$count, disp=$disp, r=$reason,";
983 | #print "dkim=$dkim/$dkimresult, spf=$spf/$spfresult\n";
984 |
985 | # What type of IP address?
986 | my ($nip, $iptype, $ipval);
987 | if ($debug) {
988 | print "ip=$ip\n";
989 | }
990 | if($nip = inet_pton(AF_INET, $ip)) {
991 | $ipval = unpack "N", $nip;
992 | $iptype = "ip";
993 | } elsif($nip = inet_pton(AF_INET6, $ip)) {
994 | $ipval = $dbx{to_hex_string}($nip);
995 | $iptype = "ip6";
996 | } else {
997 | warn "$scriptname: $org: $id: ??? mystery ip $ip\n";
998 | rollback($dbh);
999 | return 0;
1000 | }
1001 |
1002 | $dbh->do(qq{INSERT INTO rptrecord(serial,$iptype,rcount,disposition,spf_align,dkim_align,reason,dkimdomain,dkimresult,spfdomain,spfresult,identifier_hfrom)
1003 | VALUES(?,$ipval,?,?,?,?,?,?,?,?,?,?)},undef,$serial,$count,$disp,$spf_align,$dkim_align,$reason,$dkim,$dkimresult,$spf,$spfresult,$identifier_hfrom);
1004 | if ($dbh->errstr) {
1005 | warn "$scriptname: $org: $id: Cannot add report data to database. Skipped.\n";
1006 | rollback($dbh);
1007 | return 0;
1008 | }
1009 | return 1;
1010 | }
1011 |
1012 | my $res = 1;
1013 | if(ref $record eq "HASH") {
1014 | if ($debug){
1015 | print "single record\n";
1016 | }
1017 | $res = -1 if !dorow($serial,$record,$org,$id);
1018 | } elsif(ref $record eq "ARRAY") {
1019 | if ($debug){
1020 | print "multi record\n";
1021 | }
1022 | foreach my $row (@$record) {
1023 | $res = -1 if !dorow($serial,$row,$org,$id);
1024 | }
1025 | } else {
1026 | warn "$scriptname: $org: $id: mystery type " . ref($record) . "\n";
1027 | }
1028 |
1029 | if ($debug && $res <= 0) {
1030 | print "Result $res XML: $xml->{raw_xml}\n";
1031 | }
1032 |
1033 | if ($res <= 0) {
1034 | if ($db_tx_support) {
1035 | warn "$scriptname: $org: $id: Cannot add records to rptrecord. Rolling back DB transaction.\n";
1036 | rollback($dbh);
1037 | } else {
1038 | warn "$scriptname: $org: $id: errors while adding to rptrecord, serial $serial records likely obsolete.\n";
1039 | }
1040 | } else {
1041 | if ($db_tx_support) {
1042 | $dbh->commit;
1043 | if ($dbh->errstr) {
1044 | warn "$scriptname: $org: $id: Cannot commit transaction.\n";
1045 | }
1046 | }
1047 | }
1048 | return $res;
1049 | }
1050 |
1051 | ################################################################################
1052 |
1053 | # Tries to roll back the transaction (if enabled).
1054 | # If an error happens, warn the user, but continue execution.
1055 | sub rollback {
1056 | my $dbh = $_[0];
1057 |
1058 | if ($db_tx_support) {
1059 | $dbh->rollback;
1060 | if ($dbh->errstr) {
1061 | warn "$scriptname: Cannot rollback transaction.\n";
1062 | }
1063 | }
1064 | }
1065 |
1066 | ################################################################################
1067 |
1068 | # Check, if the database contains needed tables and columns. The idea is, that
1069 | # the user only has to create the database/database_user. All needed tables and
1070 | # columns are created automatically. Furthermore, if new columns are introduced,
1071 | # the user does not need to make any changes to the database himself.
1072 | sub checkDatabase {
1073 | my $dbh = $_[0];
1074 |
1075 | my $tables = $dbx{tables};
1076 |
1077 | # Create missing tables and missing columns.
1078 | for my $table ( keys %{$tables} ) {
1079 |
1080 | if (!db_tbl_exists($dbh, $table)) {
1081 |
1082 | # Table does not exist, build CREATE TABLE cmd from tables hash.
1083 | print "$scriptname: Adding missing table <" . $table . "> to the database.\n";
1084 | my $sql_create_table = "CREATE TABLE " . $table . " (\n";
1085 | for (my $i=0; $i <= $#{$tables->{$table}{"column_definitions"}}; $i+=3) {
1086 | my $col_name = $tables->{$table}{"column_definitions"}[$i];
1087 | my $col_type = $tables->{$table}{"column_definitions"}[$i+1];
1088 | my $col_opts = $tables->{$table}{"column_definitions"}[$i+2];
1089 | # add comma if second or later entry
1090 | if ($i != 0) {
1091 | $sql_create_table .= ",\n";
1092 | }
1093 | $sql_create_table .= "$col_name $col_type $col_opts";
1094 | }
1095 | # Add additional_definitions, if defined.
1096 | if ($tables->{$table}{"additional_definitions"} ne "") {
1097 | $sql_create_table .= ",\n" . $tables->{$table}{"additional_definitions"};
1098 | }
1099 | # Add options.
1100 | $sql_create_table .= ") " . $tables->{$table}{"table_options"} . ";";
1101 | # Create table.
1102 | print "$sql_create_table\n" if $debug;
1103 | $dbh->do($sql_create_table);
1104 |
1105 | # Create indexes.
1106 | foreach my $sql_idx (@{$tables->{$table}{indexes}}) {
1107 | print "$sql_idx\n" if $debug;
1108 | $dbh->do($sql_idx);
1109 | }
1110 | } else {
1111 |
1112 | #Table exists, get current columns in this table from DB.
1113 | my %db_col_exists = db_column_info($dbh, $table);
1114 |
1115 | # Check if all needed columns are present, if not add them at the desired position.
1116 | my $insert_pos;
1117 | for (my $i=0; $i <= $#{$tables->{$table}{"column_definitions"}}; $i+=3) {
1118 | my $col_name = $tables->{$table}{"column_definitions"}[$i];
1119 | my $col_type = $tables->{$table}{"column_definitions"}[$i+1];
1120 | my $col_opts = $tables->{$table}{"column_definitions"}[$i+2];
1121 | if (!$db_col_exists{$col_name}) {
1122 | # add column
1123 | my $sql_add_column = $dbx{add_column}($table, $col_name, $col_type, $col_opts, $insert_pos);
1124 | print "$sql_add_column\n" if $debug;
1125 | $dbh->do($sql_add_column);
1126 | } elsif ($db_col_exists{$col_name} !~ /^\Q$col_type\E/) {
1127 | # modify column
1128 | my $sql_modify_column = $dbx{modify_column}($table, $col_name, $col_type, $col_opts);
1129 | print "$sql_modify_column\n" if $debug;
1130 | $dbh->do($sql_modify_column);
1131 | }
1132 | $insert_pos = $col_name;
1133 | }
1134 | }
1135 | }
1136 |
1137 | $dbh->commit;
1138 | }
1139 |
1140 | ################################################################################
1141 |
1142 | # Checks if the table exists in the database
1143 | sub db_tbl_exists {
1144 | my ($dbh, $table) = @_;
1145 |
1146 | my @res = $dbh->tables(undef, undef, $table, undef);
1147 | return scalar @res > 0;
1148 | }
1149 |
1150 | ################################################################################
1151 |
1152 | # Gets columns and their data types in a given table
1153 | sub db_column_info {
1154 | my ($dbh, $table) = @_;
1155 |
1156 | my $db_info = $dbh->column_info(undef, undef, $table, undef)->fetchall_hashref('COLUMN_NAME');
1157 | my %columns;
1158 | foreach my $column (keys(%$db_info)) {
1159 | $columns{$column} = $db_info->{$column}{$dbx{column_info_type_col}};
1160 | }
1161 | return %columns;
1162 | }
1163 |
--------------------------------------------------------------------------------