├── .gitattributes ├── MANIFEST.in ├── Makefile ├── README ├── README.CREDITS ├── README.EXAMPLES ├── README.INSTALL ├── README.LICENSE ├── doc ├── giles.1 ├── user-manual.pdf └── user-manual.tex ├── examples ├── ancestry │ ├── README │ ├── ancestry.yml │ ├── input.sql │ └── output.sql ├── family │ ├── README │ ├── family.yml │ ├── input.sql │ └── output.sql ├── pets │ ├── README │ ├── input.sql │ ├── output.sql │ └── pets.yml └── tarnis │ ├── README │ ├── example-network.sql │ └── tarnis.yml ├── giles ├── __init__.py ├── caseless_string.py ├── expression.py ├── forbidden_names.py ├── giles.py ├── pyre.py ├── sqlite.jinja ├── sqlite_backend.py └── validate.py ├── setup.py ├── tests └── __init__.py └── utils ├── version2string └── version_helper /.gitattributes: -------------------------------------------------------------------------------- 1 | utils/version2string -ident 2 | utils/version_helper -ident 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Makefile 2 | include README.* 3 | include doc/*.tex 4 | include doc/*.pdf 5 | include doc/*.1 6 | include tests/*.py 7 | recursive-include utils version* 8 | recursive-include examples *.yml *.sql README 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ###################################################################### 2 | # 3 | # $Id$ 4 | # 5 | ###################################################################### 6 | # 7 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 8 | # 9 | # This software, having been partly or wholly developed and/or 10 | # sponsored by KoreLogic, Inc., is hereby released under the terms 11 | # and conditions set forth in the project's "README.LICENSE" file. 12 | # For a list of all contributors and sponsors, please refer to the 13 | # project's "README.CREDITS" file. 14 | # 15 | ###################################################################### 16 | # 17 | # Purpose: Build the Giles compiler. 18 | # 19 | ###################################################################### 20 | 21 | DESTDIR = 22 | PREFIX = /usr/local 23 | MANPATH = $(DESTDIR)$(PREFIX)/man 24 | 25 | all: build 26 | 27 | build: 28 | @python3 setup.py build 29 | 30 | clean: 31 | @python3 setup.py clean --all 32 | @rm -rf dist giles.egg-info giles/__pycache__ tests/__pycache__ examples/*/*.yml.sql 33 | 34 | dist sdist: 35 | @python3 setup.py sdist 36 | 37 | install: build 38 | @python3 setup.py install --root "/$(DESTDIR)" 39 | @mkdir -p "$(MANPATH)/man1" && install -m 0644 doc/giles.1 "$(MANPATH)/man1/giles.1" 40 | 41 | check test tests: build 42 | @python3 setup.py test 43 | 44 | check-clean: clean 45 | 46 | test-clean: clean 47 | 48 | lint flake: 49 | @which flake8 > /dev/null 50 | @python3 `which flake8` -v --max-line-length=140 setup.py 51 | @cd giles && python3 `which flake8` -v --max-line-length=140 *.py 52 | 53 | sign: dist 54 | @version_number=`egrep '^version = 0x' giles/__init__.py | awk '{print $$3}'` ; \ 55 | version_string=`utils/version2string -t tar -v $${version_number}` ; \ 56 | dist_file="dist/giles-$${version_string}.tar.gz" ; \ 57 | gpg --default-key giles-project@korelogic.com -s -b $${dist_file} 58 | 59 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | OVERVIEW 7 | 8 | Giles is a compiler for production systems. Production systems 9 | are useful computational systems for such domains as event 10 | correlation engines, expert systems, and action-selection 11 | mechanisms. Giles is unique in that it turns a normal relational 12 | database into a production system without any additional software 13 | required at runtime. Given a description of a production system 14 | in a domain specific language, Giles will produce a schema for a 15 | relational database management system. That schema will create 16 | a database that is completely self-contained and implements the 17 | described production system efficiently and simply. This makes 18 | production systems accessible to far more programmers in far more 19 | situations by allowing programmers to use preexisting database 20 | engines and libraries. 21 | 22 | The Giles Project is hosted at: 23 | 24 | https://git.korelogic.com/giles.git/ 25 | 26 | DOCUMENTATION 27 | 28 | General documentation is located in the docs directory. See the 29 | README.INSTALL file for instructions on how to build, test, and 30 | install the framework. 31 | 32 | LICENSE 33 | 34 | The terms and conditions under which this software is released are 35 | set forth in README.LICENSE. 36 | 37 | -------------------------------------------------------------------------------- /README.CREDITS: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | CREDITS 7 | 8 | Andy Bair (contributor) 9 | Derek Brown (contributor) 10 | Hank Leininger (contributor) 11 | Klayton Monroe (contributor) 12 | Kyle Wayman (contributor) 13 | Rob King (author, maintainer) 14 | 15 | SPONSORS 16 | 17 | KoreLogic (2011-Present) 18 | DARPA/AFRL Cyber Insider (CINDER) Program (2011-2014) 19 | 20 | -------------------------------------------------------------------------------- /README.EXAMPLES: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | OVERVIEW 7 | 8 | Giles ships with several examples in the examples directory. 9 | Each example has its own README file with further details. 10 | 11 | SQLITE CONFIGURATION 12 | 13 | The included examples all require SQLite 3.7.16.1 or greater, with foreign 14 | key and recursive trigger support enabled. In the SQLite command-line 15 | application, these features can be enabled with the following pragmas: 16 | 17 | PRAGMA recursive_triggers = ON; 18 | PRAGMA foreign_keys = ON; 19 | 20 | These statements can be placed in the ".sqliterc" initialization file 21 | as well. When accessing SQLite databases programmatically, refer to the 22 | access library's documentation to determine how to enable these features. 23 | 24 | LICENSE 25 | 26 | The terms and conditions under which this software is released are 27 | set forth in README.LICENSE. 28 | 29 | -------------------------------------------------------------------------------- /README.INSTALL: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | OVERVIEW 7 | 8 | Giles is a compiler for production systems. Production systems 9 | are useful computational systems for such domains as event 10 | correlation engines, expert systems, and action-selection 11 | mechanisms. Giles is unique in that it turns a normal relational 12 | database into a production system without any additional software 13 | required at runtime. Given a description of a production system 14 | in a domain specific language, Giles will produce a schema for a 15 | relational database management system. That schema will create 16 | a database that is completely self-contained and implements the 17 | described production system efficiently and simply. This makes 18 | production systems accessible to far more programmers in far more 19 | situations by allowing programmers to use preexisting database 20 | engines and libraries. 21 | 22 | The Giles Project is hosted at: 23 | 24 | https://www.korelogic.com 25 | 26 | TECHNICAL REQUIREMENTS 27 | 28 | The following software must be installed for Giles to work properly: 29 | 30 | - Python 3.4 or greater 31 | 32 | The database schemas produced by Giles have specific requirements as well. 33 | 34 | The Giles SQLite backend produces schemas that have the following requirements: 35 | 36 | - SQLite 3.7.16.1 or greater 37 | -- With foreign key support enabled 38 | -- With trigger support enabled 39 | 40 | Giles can compile production systems that use regular expressions 41 | and/or recursive productions; to use these features, the target 42 | SQLite database must provide an implementation of the REGEXP 43 | operator and/or enable recursive triggers. 44 | 45 | Other Giles backends have differing requirements; see the 46 | backend-specific documentation. 47 | 48 | Note that Giles can compile and produce schemas for database systems 49 | that are not installed on the system; the database systems are only 50 | required to run the created schemas, not to generate them. 51 | 52 | PREREQUISITES INSTALLATION 53 | 54 | Building Giles requires the following software to be installed: 55 | 56 | - Python 3.4 or greater (also required at runtime) 57 | 58 | The installation mechanism uses the Setuptools system which will 59 | automatically check for any needed dependencies. 60 | 61 | TESTING 62 | 63 | Giles provides a simple test suite that can be invoked by: 64 | 65 | $ make test 66 | 67 | INSTALLATION 68 | 69 | The easiest way to install Giles is: 70 | 71 | $ sudo make install 72 | 73 | This will install the package into the default binary directory. 74 | The Makefile supports the conventional "PREFIX" variable that 75 | specifies the top-level installation directory (by default, 76 | "/usr/local"). The Makefile also provides a "DESTDIR" variable 77 | for its "install" target to support installation to alternate 78 | roots. 79 | 80 | RUNNING GILES 81 | 82 | Giles can be run using the 'giles' command. Invoking the 'giles' 83 | command without arguments will produce a simple usage message. 84 | 85 | For more information on invoking Giles, see giles(1). 86 | -------------------------------------------------------------------------------- /doc/giles.1: -------------------------------------------------------------------------------- 1 | .Dd $Mdocdate$ 2 | .Dt GILES 1 3 | .Sh NAME 4 | .Nm giles 5 | .Nd compile production systems 6 | .Sh SYNOPSIS 7 | .Nm 8 | .Op Fl h 9 | .Op Fl v 10 | .Op Fl b Ar BACKEND 11 | .Op Fl c 12 | .Op Fl r 13 | .Op Fl p Ar PREFIX 14 | .Op Fl o Ar OUTPUT 15 | .Ar FILE 16 | .Op "FILE ..." 17 | .Sh DESCRIPTION 18 | The 19 | .Nm 20 | tool compiles a production system, producing an output SQL schema given a description in a YAML file. 21 | 22 | The following options are supported: 23 | .Bl -tag 24 | .It Fl h 25 | Display a usage message and exit. 26 | Included in this help message is a list of supported database targets. 27 | .It Fl v 28 | Display the version of the compiler and exit. 29 | .It Fl c 30 | Allow the engine to have match-assert cycles. 31 | .It Fl r 32 | Allow the engine to use regular expressions. 33 | .It Fl b Ar BACKEND 34 | Specify that the compiler should generate code using 35 | .Ar BACKEND "." 36 | .It Fl p Ar PREFIX 37 | Specify that the compiler should prefix all generated database objects with 38 | .Ar PREFIX "." 39 | By default this is 40 | .Dq "Giles" "." 41 | .It Fl o Ar OUTPUT 42 | Direct output to the named file. 43 | By default this is 44 | .Pa stdout "." 45 | .El 46 | .Pp 47 | Note that enabling regular expression or cycle support might mean enabling non-default features on the target database, and may not be supported at all in some systems. 48 | .Sh EXIT STATUS 49 | .Ex -std 50 | .Sh SEE ALSO 51 | .Rs 52 | .%A Rob King 53 | .%A Derek Brown 54 | .%B The Giles Production System Compiler User Manual 55 | .Re 56 | .Sh HISTORY 57 | .Nm 58 | first appeared publicly in 2014. 59 | .Sh AUTHORS 60 | .Nm 61 | was developed by the KoreLogic Development Team. 62 | -------------------------------------------------------------------------------- /doc/user-manual.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KoreLogicSecurity/giles/a6ae810802b1fc8fb1f24f4b4129449cf7afdbc8/doc/user-manual.pdf -------------------------------------------------------------------------------- /examples/ancestry/README: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | OVERVIEW 7 | 8 | This example illustrates a simple production system with a recursive 9 | rule. The rule infers ancestor-descendant relationships. 10 | 11 | Note that this engine can rapidly grow the size of the database for 12 | deep hierarchies. 13 | 14 | COMPILING THE ENGINE 15 | 16 | After installing Giles, this engine can be compiled using the following 17 | command: 18 | 19 | $ giles -c -o ancestry.sql ancestry.yml 20 | 21 | Note the use of the '-c' flag, indicating that the user knows that a 22 | cycle exists in the rule set. 23 | 24 | LOADING THE ENGINE 25 | 26 | The engine can be loaded into a SQLite instance with the following commands: 27 | 28 | $ sqlite3 29 | sqlite> PRAGMA recursive_triggers = ON; /* If necessary. */ 30 | sqlite> PRAGMA foreign_keys = ON; /* If necessary. */ 31 | sqlite> .read ancestry.sql 32 | 33 | USING THE ENGINE 34 | 35 | This engine describes a single class of facts, called 'IsAncestor'. 36 | This describes an ancestor-descendant relationship. 37 | 38 | Asserting known ancestor-descendant relationships will build up a 39 | working fact set: 40 | 41 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Betsy', 'James'); 42 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Betsy', 'Marvin'); 43 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Rob', 'James'); 44 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Rob', 'Marvin'); 45 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Julie', 'Betsy'); 46 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Bob', 'Betsy'); 47 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Ann', 'Rob'); 48 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Jim', 'Rob'); 49 | 50 | These statements are also available in the input.sql file that can be read 51 | directly by sqlite. 52 | 53 | The engine will automatically infer the ancestor-descendant relationship for 54 | all known people: 55 | 56 | SELECT Ancestor AS Ancestors FROM Giles_IsAncestor_Facts WHERE Descendant = 'Marvin'; 57 | 58 | Ancestors 59 | --------- 60 | Betsy 61 | Rob 62 | Julie 63 | Bob 64 | Ann 65 | Jim 66 | 67 | This selection statement is also available in the output.sql file that can 68 | be read directly by sqlite. 69 | 70 | LICENSE 71 | 72 | The terms and conditions under which this software is released are 73 | set forth in README.LICENSE. 74 | 75 | -------------------------------------------------------------------------------- /examples/ancestry/ancestry.yml: -------------------------------------------------------------------------------- 1 | ###################################################################### 2 | # 3 | # $Id$ 4 | # 5 | ###################################################################### 6 | # 7 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 8 | # 9 | # This software, having been partly or wholly developed and/or 10 | # sponsored by KoreLogic, Inc., is hereby released under the terms 11 | # and conditions set forth in the project's "README.LICENSE" file. 12 | # For a list of all contributors and sponsors, please refer to the 13 | # project's "README.CREDITS" file. 14 | # 15 | ###################################################################### 16 | # 17 | # Purpose: An example production system. 18 | # 19 | ###################################################################### 20 | 21 | Facts: 22 | IsAncestor: 23 | Descendant: STRING 24 | Ancestor: STRING 25 | 26 | Rules: 27 | AIsAncestorOfB: 28 | Description: A is an ancestor of B. 29 | 30 | MatchAll: 31 | - Fact: IsAncestor 32 | Meaning: An ancestor relationship exists. 33 | Assign: 34 | Descendant: !expr This.Descendant 35 | Ancestor: !expr This.Ancestor 36 | 37 | - Fact: IsAncestor 38 | Meaning: That ancestor has an ancestor. 39 | When: !expr This.Descendant == Locals.Ancestor 40 | Assign: 41 | Grandparent: !expr This.Ancestor 42 | 43 | Assert: !distinct 44 | IsAncestor: 45 | Descendant: !expr Locals.Descendant 46 | Ancestor: !expr Locals.Grandparent 47 | -------------------------------------------------------------------------------- /examples/ancestry/input.sql: -------------------------------------------------------------------------------- 1 | PRAGMA foreign_keys = 1; 2 | PRAGMA recursive_triggers = 1; 3 | 4 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Betsy', 'James'); 5 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Betsy', 'Marvin'); 6 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Rob', 'James'); 7 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Rob', 'Marvin'); 8 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Julie', 'Betsy'); 9 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Bob', 'Betsy'); 10 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Ann', 'Rob'); 11 | INSERT INTO Giles_IsAncestor_Facts(Ancestor, Descendant) VALUES('Jim', 'Rob'); 12 | 13 | -------------------------------------------------------------------------------- /examples/ancestry/output.sql: -------------------------------------------------------------------------------- 1 | PRAGMA foreign_keys = 1; 2 | PRAGMA recursive_triggers = 1; 3 | 4 | SELECT Ancestor FROM Giles_IsAncestor_Facts WHERE Descendant = 'Marvin'; 5 | -------------------------------------------------------------------------------- /examples/family/README: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | OVERVIEW 7 | 8 | This example illustrates a simple production system that, when given 9 | facts about individuals and families, infers to which families individuals 10 | may belong. This example is of course contrived, but illustrates basic 11 | engine features, and especially highlights the foreign function interface. 12 | 13 | COMPILING THE ENGINE 14 | 15 | After installing Giles, this engine can be compiled using the following 16 | command: 17 | 18 | $ giles -o family.sql family.yml 19 | 20 | LOADING THE ENGINE 21 | 22 | The engine can be loaded into a SQLite instance with the following commands: 23 | 24 | $ sqlite3 25 | sqlite> PRAGMA recursive_triggers = ON; /* If necessary. */ 26 | sqlite> PRAGMA foreign_keys = ON; /* If necessary. */ 27 | sqlite> .read family.sql 28 | 29 | USING THE ENGINE 30 | 31 | This engine describes four classes of facts. 32 | 33 | Each known family has a fact listing its family name: 34 | 35 | INSERT INTO Giles_KnownFamilies_Facts(Family) VALUES('Jones'); 36 | INSERT INTO Giles_KnownFamilies_Facts(Family) VALUES('King'); 37 | INSERT INTO Giles_KnownFamilies_Facts(Family) VALUES('Smith'); 38 | 39 | Each individual person has a fact giving his or her first and last names: 40 | 41 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('Rob', 'King'); 42 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('Betsy', 'King'); 43 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('John', 'Smith'); 44 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('John', 'Jones'); 45 | 46 | These statements are also available in the input.sql file that can be read 47 | directly by sqlite. 48 | 49 | The engine will automatically produce MightBelongToFamily facts that 50 | indicate which individuals might belong to which families: 51 | 52 | sqlite> SELECT Family, FirstName FROM Giles_MightBelongToFamily_Facts; 53 | Family FirstName 54 | ---------- ---------- 55 | King Rob 56 | King Betsy 57 | Smith John 58 | Jones John 59 | 60 | This statement is also available in the output.sql file that can be read 61 | directly by sqlite. 62 | 63 | An explicit exception can be made for any individual by asserting a 64 | KnownException fact: 65 | 66 | sqlite> INSERT INTO Giles_KnownException_Facts(FirstName, Family) VALUES('John', 'Smith'); 67 | sqlite> SELECT Family, FirstName FROM Giles_MightBelongToFamily_Facts; 68 | Family FirstName 69 | ---------- ---------- 70 | King Rob 71 | King Betsy 72 | Jones John 73 | 74 | Removing the exception will of course restore the original inference: 75 | 76 | sqlite> DELETE FROM Giles_KnownException_Facts; 77 | sqlite> SELECT Family, FirstName FROM Giles_MightBelongToFamily_Facts; 78 | Family FirstName 79 | ---------- ---------- 80 | King Rob 81 | King Betsy 82 | Jones John 83 | Smith John 84 | 85 | LICENSE 86 | 87 | The terms and conditions under which this software is released are 88 | set forth in README.LICENSE. 89 | 90 | -------------------------------------------------------------------------------- /examples/family/family.yml: -------------------------------------------------------------------------------- 1 | ###################################################################### 2 | # 3 | # $Id$ 4 | # 5 | ###################################################################### 6 | # 7 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 8 | # 9 | # This software, having been partly or wholly developed and/or 10 | # sponsored by KoreLogic, Inc., is hereby released under the terms 11 | # and conditions set forth in the project's "README.LICENSE" file. 12 | # For a list of all contributors and sponsors, please refer to the 13 | # project's "README.CREDITS" file. 14 | # 15 | ###################################################################### 16 | # 17 | # Purpose: An example production system. 18 | # 19 | ###################################################################### 20 | 21 | Functions: 22 | Lowercase: 23 | External: LOWER 24 | Parameters: [STRING] 25 | Returns: STRING 26 | 27 | Uppercase: 28 | External: UPPER 29 | Parameters: [STRING] 30 | Returns: STRING 31 | 32 | Strlen: 33 | External: LENGTH 34 | Parameters: [STRING] 35 | Returns: INTEGER 36 | 37 | Substr: 38 | External: SUBSTR 39 | Parameters: [STRING, INTEGER, INTEGER] 40 | Returns: STRING 41 | 42 | Facts: 43 | KnownFamilies: 44 | Family: STRING 45 | 46 | PersonIsNamed: 47 | FirstName: STRING 48 | LastName: STRING 49 | 50 | MightBelongToFamily: 51 | FirstName: STRING 52 | Family: STRING 53 | 54 | KnownException: 55 | FirstName: STRING 56 | Family: STRING 57 | 58 | Rules: 59 | PersonMightBelongToFamily: 60 | Description: A named person might belong to a known family. 61 | 62 | MatchAll: 63 | - Fact: PersonIsNamed 64 | Meaning: A person has been named. 65 | Assign: 66 | FirstName: !expr This.FirstName 67 | LastName: !expr This.LastName 68 | 69 | - Fact: KnownFamilies 70 | Meaning: A family with a matching name exists. 71 | When: !expr This.Family == UPPERCASE(SUBSTR(Locals.LastName, 1, 1)) 72 | . 73 | LOWERCASE(SUBSTR(Locals.LastName, 2, STRLEN(Locals.LastName))) 74 | Assign: 75 | Family: !expr This.Family 76 | 77 | MatchNone: 78 | - Fact: KnownException 79 | Meaning: We know that a particular person isn't a member of a certain family (uppercase). 80 | When: !expr This.FirstName == UPPERCASE(Locals.FirstName) AND 81 | This.Family == UPPERCASE(Locals.Family) 82 | 83 | - Fact: KnownException 84 | Meaning: We know that a particular person isn't a member of a certain family (lowercase). 85 | When: !expr This.FirstName == LOWERCASE(Locals.FirstName) AND 86 | This.Family == LOWERCASE(Locals.Family) 87 | 88 | - Fact: KnownException 89 | Meaning: We know that a particular person isn't a member of a certain family (titlecase). 90 | When: !expr This.FirstName == UPPERCASE(SUBSTR(Locals.FirstName, 1, 1)) 91 | . 92 | LOWERCASE(SUBSTR(Locals.FirstName, 2, STRLEN(Locals.FirstName))) 93 | AND 94 | This.Family == UPPERCASE(SUBSTR(Locals.LastName, 1, 1)) 95 | . 96 | LOWERCASE(SUBSTR(Locals.LastName, 2, STRLEN(Locals.LastName))) 97 | 98 | Assert: 99 | MightBelongToFamily: 100 | FirstName: !expr Locals.FirstName 101 | Family: !expr Locals.Family 102 | -------------------------------------------------------------------------------- /examples/family/input.sql: -------------------------------------------------------------------------------- 1 | PRAGMA foreign_keys = 1; 2 | PRAGMA recursive_triggers = 1; 3 | 4 | INSERT INTO Giles_KnownFamilies_Facts(Family) VALUES('Jones'); 5 | INSERT INTO Giles_KnownFamilies_Facts(Family) VALUES('King'); 6 | INSERT INTO Giles_KnownFamilies_Facts(Family) VALUES('Smith'); 7 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('Rob', 'King'); 8 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('Betsy', 'King'); 9 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('John', 'Smith'); 10 | INSERT INTO Giles_PersonIsNamed_Facts(FirstName, LastName) VALUES('John', 'Jones'); 11 | -------------------------------------------------------------------------------- /examples/family/output.sql: -------------------------------------------------------------------------------- 1 | PRAGMA foreign_keys = 1; 2 | PRAGMA recursive_triggers = 1; 3 | 4 | SELECT Family, FirstName FROM Giles_MightBelongToFamily_Facts; 5 | -------------------------------------------------------------------------------- /examples/pets/README: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | OVERVIEW 7 | 8 | This example illustrates a simple production system that infers that an 9 | animal living in the same domicile as a human is a pet of that human. 10 | 11 | COMPILING THE ENGINE 12 | 13 | After installing Giles, this engine can be compiled using the following 14 | command: 15 | 16 | $ giles -o pets.sql pets.yml 17 | 18 | LOADING THE ENGINE 19 | 20 | The engine can be loaded into a SQLite instance with the following commands: 21 | 22 | $ sqlite3 23 | sqlite> PRAGMA recursive_triggers = ON; /* If necessary. */ 24 | sqlite> PRAGMA foreign_keys = ON; /* If necessary. */ 25 | sqlite> .read pets.sql 26 | 27 | USING THE ENGINE 28 | 29 | This engine describes four classes of facts. 30 | 31 | Each known person has a fact asserting his or her existence (the Age field 32 | is not used in this engine, but appears as an illustrative example to show 33 | that not all fields need to be used in a rule): 34 | 35 | INSERT INTO Giles_PersonExists_Facts(Name, Age) VALUES('Rob', 34); 36 | INSERT INTO Giles_PersonExists_Facts(Name, Age) VALUES('Joel', 30); 37 | 38 | Each animal likewise has a fact asserting his or her existence (the Age 39 | field is again not used): 40 | 41 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Caboose', 6); 42 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Donut', 11); 43 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Vanna', 12); 44 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Leo', 7); 45 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('June', 2); 46 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Franny', 8); 47 | 48 | Each person or animal can be declared to live in some particular domicile: 49 | 50 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Rob', 'Chez Rob'); 51 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Joel', 'Casa del Joel'); 52 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Caboose', 'Chez Rob'); 53 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Donut', 'Chez Rob'); 54 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Vanna', 'Chez Rob'); 55 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('June', 'Casa del Joel'); 56 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Leo', 'Casa del Joel'); 57 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Franny', 'Casa del Joel'); 58 | 59 | These statements are also available in the input.sql file that can be read 60 | directly by sqlite. 61 | 62 | The engine will automatically produce IsAPet facts for the animals that 63 | live with people: 64 | 65 | SELECT Person, Pet FROM Giles_IsAPet_Facts; 66 | Person Pet 67 | ---------- ---------- 68 | Rob Caboose 69 | Rob Donut 70 | Rob Vanna 71 | Joel June 72 | Joel Leo 73 | Joel Franny 74 | 75 | This statement is also available in the output.sql file that can be read 76 | directly by sqlite. 77 | 78 | LICENSE 79 | 80 | The terms and conditions under which this software is released are 81 | set forth in README.LICENSE. 82 | 83 | -------------------------------------------------------------------------------- /examples/pets/input.sql: -------------------------------------------------------------------------------- 1 | PRAGMA foreign_keys = 1; 2 | PRAGMA recursive_triggers = 1; 3 | 4 | INSERT INTO Giles_PersonExists_Facts(Name, Age) VALUES('Rob', 34); 5 | INSERT INTO Giles_PersonExists_Facts(Name, Age) VALUES('Joel', 30); 6 | 7 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Caboose', 6); 8 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Donut', 11); 9 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Vanna', 12); 10 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Leo', 7); 11 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('June', 2); 12 | INSERT INTO Giles_AnimalExists_Facts(Name, Age) VALUES('Franny', 8); 13 | 14 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Rob', 'Chez Rob'); 15 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Joel', 'Casa del Joel'); 16 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Caboose', 'Chez Rob'); 17 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Donut', 'Chez Rob'); 18 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Vanna', 'Chez Rob'); 19 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('June', 'Casa del Joel'); 20 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Leo', 'Casa del Joel'); 21 | INSERT INTO Giles_Inhabits_Facts(Name, Domicile) VALUES('Franny', 'Casa del Joel'); 22 | -------------------------------------------------------------------------------- /examples/pets/output.sql: -------------------------------------------------------------------------------- 1 | PRAGMA foreign_keys = 1; 2 | PRAGMA recursive_triggers = 1; 3 | 4 | SELECT Person, Pet FROM Giles_IsAPet_Facts; 5 | -------------------------------------------------------------------------------- /examples/pets/pets.yml: -------------------------------------------------------------------------------- 1 | ###################################################################### 2 | # 3 | # $Id$ 4 | # 5 | ###################################################################### 6 | # 7 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 8 | # 9 | # This software, having been partly or wholly developed and/or 10 | # sponsored by KoreLogic, Inc., is hereby released under the terms 11 | # and conditions set forth in the project's "README.LICENSE" file. 12 | # For a list of all contributors and sponsors, please refer to the 13 | # project's "README.CREDITS" file. 14 | # 15 | ###################################################################### 16 | # 17 | # Purpose: An example production system. 18 | # 19 | ###################################################################### 20 | 21 | Description: Figure out who owns which pets. 22 | 23 | Facts: 24 | PersonExists: 25 | Name: STRING 26 | Age: INTEGER 27 | 28 | AnimalExists: 29 | Name: STRING 30 | Age: INTEGER 31 | 32 | Inhabits: 33 | Domicile: STRING 34 | Name: STRING 35 | 36 | IsAPet: 37 | Person: STRING 38 | Pet: STRING 39 | 40 | Rules: 41 | AIsPetOfB: 42 | Description: A is the pet of B. 43 | 44 | MatchAll: 45 | - Fact: AnimalExists 46 | Meaning: An animal exists. 47 | Assign: 48 | PetName: !expr This.Name 49 | 50 | - Fact: Inhabits 51 | Meaning: The animal lives somewhere. 52 | When: !expr This.Name == Locals.PetName 53 | Assign: 54 | Domicile: !expr This.Domicile 55 | 56 | - Fact: PersonExists 57 | Meaning: A person exists. 58 | Assign: 59 | PersonName: !expr This.Name 60 | 61 | - Fact: Inhabits 62 | Meaning: The person lives in the same place as the animal. 63 | When: !expr This.Name == Locals.PersonName AND This.Domicile == Locals.Domicile 64 | 65 | Assert: 66 | IsAPet: 67 | Person: !expr Locals.PersonName 68 | Pet: !expr Locals.PetName 69 | 70 | -------------------------------------------------------------------------------- /examples/tarnis/README: -------------------------------------------------------------------------------- 1 | 2 | REVISION 3 | 4 | $Id$ 5 | 6 | OVERVIEW 7 | 8 | This example illustrates an expert system called Tarnis. Tarnis is an 9 | expert in secure network design: networks can be described by facts naming 10 | the networks and their security levels. Other facts describe uni- and 11 | bidirectional connections between various networks. Tarnis ensures that 12 | there are no reverse data flows, where data can flow from a more-secure 13 | to a less secure network by any path. 14 | 15 | Tarnis provides an interesting use case for a Giles engine. Because Giles 16 | guarantees that all derivable facts are derived and because the engine 17 | reacts dynamically, users can experiment with different network topologies 18 | by adding and removing facts and see the results of those experiments. 19 | 20 | COMPILING THE ENGINE 21 | 22 | After installing Giles, this engine can be compiled using the following 23 | command: 24 | 25 | $ giles -o tarnis.sql -r -c -p tarnis tarnis.yml 26 | 27 | The '-r' flag enables regular expression support in the compiler. The '-c' 28 | flag allows the signature set to contain cycles. 29 | 30 | Finally, the '-p' flag prefixes all of the generated database structures 31 | with 'tarnis'. This is not strictly necessary, but the example provided 32 | in example-network.sql assumes that this prefix has been used. 33 | 34 | LOADING THE ENGINE 35 | 36 | The engine can be loaded into a SQLite instance with the following commands: 37 | 38 | $ sqlite3 39 | sqlite> PRAGMA recursive_triggers = ON; /* If necessary. */ 40 | sqlite> PRAGMA foreign_keys = ON; /* If necessary. */ 41 | sqlite> .read tarnis.sql 42 | 43 | Note that SQLite must have regular expression support enabled prior 44 | to loading the compiled SQL; see SQLite's documentation for more 45 | information. 46 | 47 | USING THE ENGINE 48 | 49 | This is a fairly complex example. An additional file, example-network.sql, 50 | is provided that declares some networks and connections between them. 51 | See the comments in that file for more information. 52 | 53 | LICENSE 54 | 55 | The terms and conditions under which this software is released are 56 | set forth in README.LICENSE. 57 | 58 | -------------------------------------------------------------------------------- /examples/tarnis/example-network.sql: -------------------------------------------------------------------------------- 1 | /********************************************************************** 2 | * 3 | * $Id$ 4 | * 5 | ********************************************************************** 6 | * 7 | * Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 8 | * 9 | * This software, having been partly or wholly developed and/or 10 | * sponsored by KoreLogic, Inc., is hereby released under the terms 11 | * and conditions set forth in the project's "README.LICENSE" file. 12 | * For a list of all contributors and sponsors, please refer to the 13 | * project's "README.CREDITS" file. 14 | * 15 | ********************************************************************** 16 | * 17 | * Purpose: Tarnis is an example expert system. 18 | * 19 | ********************************************************************** 20 | */ 21 | 22 | /* Name our networks and assign each of them a security level. Note that we don't declare Network 1, we let Tarnis infer its existence. */ 23 | INSERT INTO Tarnis_NetworkExists_facts(NetworkName, SecurityLevel) VALUES('Network 2', 20); 24 | INSERT INTO Tarnis_NetworkExists_facts(NetworkName, SecurityLevel) VALUES('Network 3', 30); 25 | INSERT INTO Tarnis_NetworkExists_facts(NetworkName, SecurityLevel) VALUES('Network 4', 40); 26 | INSERT INTO Tarnis_NetworkExists_facts(NetworkName, SecurityLevel) VALUES('Network 5', 50); 27 | INSERT INTO Tarnis_NetworkExists_facts(NetworkName, SecurityLevel) VALUES('Network 6', 60); 28 | 29 | /* Declare the network topology. All of the links are unidirectional, with information only flowing from less-secure to more-secure networks... */ 30 | INSERT INTO Tarnis_UnidirectionalNetworkConnection_facts(NetworkA, NetworkB) VALUES('Network 1', 'Network 2'); 31 | INSERT INTO Tarnis_UnidirectionalNetworkConnection_facts(NetworkA, NetworkB) VALUES('Network 2', 'Network 3'); 32 | INSERT INTO Tarnis_UnidirectionalNetworkConnection_facts(NetworkA, NetworkB) VALUES('Network 3', 'Network 4'); 33 | INSERT INTO Tarnis_UnidirectionalNetworkConnection_facts(NetworkA, NetworkB) VALUES('Network 4', 'Network 5'); 34 | 35 | /* ..except for a bidirectional link which allows backflow from network 5 to network 3, and therefore from 4 to 3 (by 4 -> 5 -> 3). */ 36 | INSERT INTO Tarnis_BidirectionalNetworkConnection_facts(NetworkA, NetworkB) VALUES('Network 3', 'Network 5'); 37 | 38 | /* This backflow is documented and exceptions were allowed. In other words, we want to tell Tarnis that we're aware of 39 | * traffic flowing from Network 5 to Network 3, and from Network 4 to Network 3, and that we are explicitly okay with it. */ 40 | INSERT INTO Tarnis_AllowException_facts(NetworkA, NetworkB) VALUES('Network 5', 'Network 3'); 41 | INSERT INTO Tarnis_AllowException_facts(NetworkA, NetworkB) VALUES('Network 4', 'Network 3'); 42 | 43 | /* But did you see the flaw? We've now allowed data to flow from 5 to 4, via the 5 -> 3 -> 4 path. This wasn't documented or explicitly allowed! 44 | * Let's see if Tarnis notices the problem and fires an alert: 45 | */ 46 | 47 | SELECT * FROM Tarnis_Alert_facts; 48 | 49 | /* We also included Network 6 up there with no connections to anyone else, just to provide an "orphan network" alert. 50 | * Read tarnis.yml to see all of the different things we check for. 51 | * 52 | * What makes Tarnis such a good demonstration of Giles is that you can add or remove networks, bidirectional or unidirectional channels, exceptions, 53 | * and so on, and the system automatically maintains a list of alerts for issues that need to be addrressed. Alerts will go away if the problem goes 54 | * away or is remedied (via an exception for example), and will come back if the problem comes back. 55 | * 56 | * Try different remedies - add an explicit exception, break the link between 5 and 3 and so on. 57 | * 58 | * Tarnis, thanks to Giles, also can justify any event, including alerts. 59 | * We could justify an alert by asking: SELECT justification FROM Tarnis_Alert_justification WHERE id = 60 | * That justification will include the facts that led to that event, so you can ask for their justifications, all the way back to the 61 | * initial, user-provided facts. 62 | */ 63 | 64 | -------------------------------------------------------------------------------- /examples/tarnis/tarnis.yml: -------------------------------------------------------------------------------- 1 | ###################################################################### 2 | # 3 | # $Id: aa13c518509b431ccdebc7523def35ad4548da75 $ 4 | # 5 | ###################################################################### 6 | # 7 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 8 | # 9 | # This software, having been partly or wholly developed and/or 10 | # sponsored by KoreLogic, Inc., is hereby released under the terms 11 | # and conditions set forth in the project's "README.LICENSE" file. 12 | # For a list of all contributors and sponsors, please refer to the 13 | # project's "README.CREDITS" file. 14 | # 15 | ###################################################################### 16 | # 17 | # Purpose: Tarnis is an example expert system. 18 | # 19 | ###################################################################### 20 | 21 | Description: > 22 | This describes an expert system that knows about problems with 23 | network topologies. It can detect if traffic can flow from a 24 | more-secure to a less-secure network via any path, and can 25 | detect orphaned networks that are not connected to any other 26 | network. 27 | 28 | Parameters: 29 | InferNetworkExistence: # If true, infer that networks exist if they are mentioned in a connection description, even if they're not explicitly declared. 30 | Default: TRUE 31 | 32 | DefaultSecurityLevel: # Provides the default security level for an inferred network. 33 | Default: 0 34 | Lower: 0 35 | Upper: 100 36 | 37 | SuppressAlertPatterns: # Suppress alerts whose Message fields match any of these patterns. 38 | Dictionary: TRUE 39 | Default: '' 40 | 41 | UnsuppressAlertPatterns: # Don't suppress alerts if there is an exemption for that alert pattern. 42 | Dictionary: TRUE 43 | Default: '' 44 | 45 | Functions: 46 | Strlen: 47 | External: LENGTH 48 | Parameters: [STRING] 49 | Returns: INTEGER 50 | 51 | Facts: 52 | NetworkExists: # Input Fact - Declares that a given network exists with a given security level. 53 | NetworkName: STRING 54 | SecurityLevel: INTEGER 55 | 56 | NetworkInferred: # Internal Fact - Declares that a given network exists by inference. 57 | NetworkName: STRING 58 | SecurityLevel: INTEGER 59 | 60 | NetworkDeclaration: # Internal Fact - Describes a network found via declaration or inference. 61 | NetworkName: STRING 62 | SecurityLevel: INTEGER 63 | 64 | BidirectionalNetworkConnection: # Input Fact - Declares that two networks are directly connected with data flowing between them. 65 | NetworkA: STRING 66 | NetworkB: STRING 67 | 68 | UnidirectionalNetworkConnection: # Input Fact - Declares that two networks are directly connected with data flowing only from A to B. 69 | NetworkA: STRING 70 | NetworkB: STRING 71 | 72 | AllowException: # Input Fact - Declares that information is allowed to flow from a more-secure to a less-secure network. 73 | NetworkA: STRING 74 | NetworkB: STRING 75 | 76 | AuditedConnection: # Internal Fact - Used to track both kinds of connection. 77 | NetworkA: STRING 78 | NetworkB: STRING 79 | 80 | DataFlow: # Internal Fact - Used to track data flows. 81 | NetworkA: STRING 82 | NetworkB: STRING 83 | 84 | Alert: # Output Fact - Used to let the user know something is up. 85 | Message: STRING 86 | 87 | Rules: 88 | AddExplicitNetworkDeclaration: 89 | Description: A network exists because a user declared it to exist. 90 | 91 | MatchAll: 92 | - Fact: NetworkExists 93 | Meaning: A user has declared a network to exist. 94 | Assign: 95 | NetworkName: !expr This.NetworkName 96 | SecurityLevel: !expr This.SecurityLevel 97 | 98 | Assert: 99 | NetworkDeclaration: 100 | NetworkName: !expr Locals.NetworkName 101 | SecurityLevel: !expr Locals.SecurityLevel 102 | 103 | AddInferredNetworkDeclaration: 104 | Description: A network exists because its existence was inferred. 105 | 106 | MatchAll: 107 | - Fact: NetworkInferred 108 | Meaning: A network was inferred to exist. 109 | Assign: 110 | NetworkName: !expr This.NetworkName 111 | SecurityLevel: !expr This.SecurityLevel 112 | 113 | Assert: 114 | NetworkDeclaration: 115 | NetworkName: !expr Locals.NetworkName 116 | SecurityLevel: !expr Locals.SecurityLevel 117 | 118 | AuditBidirectionalNetworkConnection: 119 | Description: A bidirectional network connection exists and should be examined. 120 | 121 | MatchAll: 122 | - Fact: BidirectionalNetworkConnection 123 | Meaning: A connection between two networks has been declared. 124 | Assign: 125 | NetworkA: !expr This.NetworkA 126 | NetworkB: !expr This.NetworkB 127 | 128 | Assert: 129 | AuditedConnection: 130 | NetworkA: !expr Locals.NetworkA 131 | NetworkB: !expr Locals.NetworkB 132 | 133 | AuditUnidirectionalNetworkConnection: 134 | Description: A unidirectional network connection exists and should be examined. 135 | 136 | MatchAll: 137 | - Fact: UnidirectionalNetworkConnection 138 | Meaning: A connection between two networks has been declared. 139 | Assign: 140 | NetworkA: !expr This.NetworkA 141 | NetworkB: !expr This.NetworkB 142 | 143 | Assert: 144 | AuditedConnection: 145 | NetworkA: !expr Locals.NetworkA 146 | NetworkB: !expr Locals.NetworkB 147 | 148 | InferNetworkExistsA: 149 | Description: The existence of a network was inferred from its mention in a connection declaration. 150 | 151 | MatchAll: 152 | - Fact: InferNetworkExistence 153 | Meaning: The system is configured to infer network existence. 154 | When: !expr This.Value == TRUE 155 | 156 | - Fact: DefaultSecurityLevel 157 | Meaning: The system is configured to provide a default security level for an inferred network. 158 | Assign: 159 | DefaultSecurityLevel: !expr This.Value 160 | 161 | - Fact: AuditedConnection 162 | Meaning: A connection exists between two networks. 163 | Assign: 164 | NetworkA: !expr This.NetworkA 165 | 166 | MatchNone: 167 | - Fact: NetworkExists 168 | Meaning: The network was not separately declared to exist. 169 | When: !expr This.NetworkName == Locals.NetworkA 170 | 171 | Assert: !distinct 172 | NetworkInferred: 173 | NetworkName: !expr Locals.NetworkA 174 | SecurityLevel: !expr Locals.DefaultSecurityLevel 175 | 176 | InferNetworkExistsB: 177 | Description: The existence of a network was inferred from its mention in a connection declaration. 178 | 179 | MatchAll: 180 | - Fact: InferNetworkExistence 181 | Meaning: The system is configured to infer network existence. 182 | When: !expr This.Value == TRUE 183 | 184 | - Fact: DefaultSecurityLevel 185 | Meaning: The system is configured to provide a default security level for an inferred network. 186 | Assign: 187 | DefaultSecurityLevel: !expr This.Value 188 | 189 | - Fact: AuditedConnection 190 | Meaning: A connection exists between two networks. 191 | Assign: 192 | NetworkB: !expr This.NetworkB 193 | 194 | MatchNone: 195 | - Fact: NetworkExists 196 | Meaning: The network was not separately declared to exist on the left-hand side of a connection. 197 | When: !expr This.NetworkName == Locals.NetworkB 198 | 199 | Assert: !distinct 200 | NetworkInferred: 201 | NetworkName: !expr Locals.NetworkB 202 | SecurityLevel: !expr Locals.DefaultSecurityLevel 203 | 204 | UnknownNetworkConnectionA: 205 | Description: A network connection exists that connects to a network that does not exist. 206 | 207 | MatchAll: 208 | - Fact: AuditedConnection 209 | Meaning: A connection between two networks has been declared. 210 | Assign: 211 | NetworkA: !expr This.NetworkA 212 | NetworkB: !expr This.NetworkB 213 | 214 | MatchNone: 215 | - Fact: NetworkExists 216 | Meaning: The left-hand network in that declaration has not been declared to exist. 217 | When: !expr This.NetworkName == Locals.NetworkA 218 | 219 | - Fact: NetworkInferred 220 | Meaning: The left-hand network in that declaration has not been inferred to exist. 221 | When: !expr This.NetworkName == Locals.NetworkA 222 | 223 | Assert: 224 | Alert: 225 | Message: !expr ( 'A connection exists for network "' . Locals.NetworkA . '" (' . Locals.NetworkA . ' -> ' . Locals.NetworkB . '), but that network does not exist.') 226 | 227 | UnknownNetworkConnectionB: 228 | Description: A network connection exists that connects to a network that does not exist. 229 | 230 | MatchAll: 231 | - Fact: AuditedConnection 232 | Meaning: A connection between two networks has been declared. 233 | Assign: 234 | NetworkA: !expr This.NetworkA 235 | NetworkB: !expr This.NetworkB 236 | 237 | MatchNone: 238 | - Fact: NetworkExists 239 | Meaning: The right-hand network in that declaration has not been declared to exist. 240 | When: !expr This.NetworkName == Locals.NetworkB 241 | 242 | - Fact: NetworkInferred 243 | Meaning: The right-hand network in that declaration has not been inferred to exist. 244 | When: !expr This.NetworkName == Locals.NetworkB 245 | 246 | Assert: 247 | Alert: 248 | Message: !expr ( 'A connection exists for network "' . Locals.NetworkB . '" (' . Locals.NetworkA . ' -> ' . Locals.NetworkB . '), but that network does not exist.') 249 | 250 | OrphanedNetwork: 251 | Description: A network exists that connects to no other network. 252 | 253 | MatchAll: 254 | - Fact: NetworkDeclaration 255 | Meaning: A network has been declared to exist. 256 | Assign: 257 | NetworkName: !expr This.NetworkName 258 | 259 | MatchNone: 260 | - Fact: AuditedConnection 261 | Meaning: The declared network does not form the left-hand side of any connection. 262 | When: !expr This.NetworkA == Locals.NetworkName 263 | 264 | - Fact: AuditedConnection 265 | Meaning: The declared network does not form the right-hand side of any connection. 266 | When: !expr This.NetworkB == Locals.NetworkName 267 | 268 | Assert: 269 | Alert: 270 | Message: !expr ( 'Network "' . Locals.NetworkName . '" exists but is connected to no other network.') 271 | 272 | LoopbackConnection: 273 | Description: A network connection exists that connects a network to itself. 274 | 275 | MatchAll: 276 | - Fact: AuditedConnection 277 | Meaning: A connection from one network to the same network has been declared. 278 | When: !expr This.NetworkA == This.NetworkB 279 | Assign: 280 | NetworkName: !expr This.NetworkA 281 | 282 | Assert: 283 | Alert: 284 | Message: !expr ( 'A connection exists that connects a network to itself (' . Locals.NetworkName . ' -> ' . Locals.NetworkName . ').') 285 | 286 | AddUnidirectionalDataFlow: 287 | Description: Data can flow between two networks in one direction. 288 | 289 | MatchAll: 290 | - Fact: UnidirectionalNetworkConnection 291 | Meaning: A connection exists between two networks, but only in one direction. 292 | Assign: 293 | NetworkA: !expr This.NetworkA 294 | NetworkB: !expr This.NetworkB 295 | 296 | Assert: !distinct 297 | DataFlow: 298 | NetworkA: !expr Locals.NetworkA 299 | NetworkB: !expr Locals.NetworkB 300 | 301 | AddBidirectionalDataFlowA: 302 | Description: Data can flow between two networks in both directions. 303 | 304 | MatchAll: 305 | - Fact: BidirectionalNetworkConnection 306 | Meaning: A connection exists between two networks. 307 | Assign: 308 | NetworkA: !expr This.NetworkA 309 | NetworkB: !expr This.NetworkB 310 | 311 | Assert: !distinct 312 | DataFlow: 313 | NetworkA: !expr Locals.NetworkA 314 | NetworkB: !expr Locals.NetworkB 315 | 316 | AddBidirectionalDataFlowB: 317 | Description: Data can flow between two networks in both directions. 318 | 319 | MatchAll: 320 | - Fact: BidirectionalNetworkConnection 321 | Meaning: A connection exists between two networks. 322 | Assign: 323 | NetworkA: !expr This.NetworkA 324 | NetworkB: !expr This.NetworkB 325 | 326 | Assert: !distinct 327 | DataFlow: 328 | NetworkA: !expr Locals.NetworkB 329 | NetworkB: !expr Locals.NetworkA 330 | 331 | AddDistantDataFlow: 332 | Description: Data can flow between two distantly-connected networks. 333 | 334 | MatchAll: 335 | - Fact: DataFlow 336 | Meaning: Network A exists and is connected to Network B. 337 | Assign: 338 | NetworkA: !expr This.NetworkA 339 | NetworkB: !expr This.NetworkB 340 | 341 | - Fact: DataFlow 342 | Meaning: Network B is connected to another network that is not Network A. 343 | When: !expr This.NetworkA == Locals.NetworkB AND This.NetworkB != Locals.NetworkA 344 | Assign: 345 | DistantNetwork: !expr This.NetworkB 346 | 347 | Assert: !distinct 348 | DataFlow: 349 | NetworkA: !expr Locals.NetworkA 350 | NetworkB: !expr Locals.DistantNetwork 351 | 352 | InsecureDataFlow: 353 | Description: Data can flow from a more-secure network to a less-secure network. 354 | 355 | MatchAll: 356 | - Fact: DataFlow 357 | Meaning: Network A can send data to Network B. 358 | Assign: 359 | NetworkA: !expr This.NetworkA 360 | NetworkB: !expr This.NetworkB 361 | 362 | - Fact: NetworkDeclaration 363 | Meaning: Network A exists and has a higher security level than Network B. 364 | When: !expr This.NetworkName == Locals.NetworkA 365 | Assign: 366 | LevelA: !expr This.SecurityLevel 367 | 368 | - Fact: NetworkDeclaration 369 | Meaning: Network B exists and has a lower security level than Network A. 370 | When: !expr This.NetworkName == Locals.NetworkB AND This.SecurityLevel < Locals.LevelA 371 | Assign: 372 | LevelB: !expr This.SecurityLevel 373 | 374 | MatchNone: 375 | - Fact: AllowException 376 | Meaning: This is not explicitly allowed via an exception. 377 | When: !expr This.NetworkA == Locals.NetworkA AND This.NetworkB == Locals.NetworkB 378 | 379 | Assert: 380 | Alert: 381 | Message: !expr ( 'Data can flow from a more-secure network (' . Locals.NetworkA . ', level ' . string_of_int(Locals.LevelA) . ') ' . 382 | 'to a less-secure network (' . Locals.NetworkB . ', level ' . string_of_int(Locals.LevelB) . ')') 383 | 384 | SuppressAlerts: 385 | Description: Suppress alerts that match the given pattern, unless there is an UnsuppressAlertPatern for it. 386 | 387 | MatchAll: 388 | - Fact: SuppressAlertPatterns 389 | Meaning: The system has been configured to supress alerts matching a certain pattern. 390 | Assign: 391 | SuppressionName: !expr This.Key 392 | Pattern: !expr This.Value 393 | PatternLength: !expr String_Of_Int(strlen(This.Value)) 394 | 395 | MatchNone: 396 | - Fact: UnsuppressAlertPatterns 397 | Meaning: The system was not configured to exempt this alert based on a different pattern. 398 | When: !expr This.Value == Locals.SuppressionName 399 | 400 | Suppress: 401 | Fact: Alert 402 | When: !expr This.Message ~ Locals.Pattern -------------------------------------------------------------------------------- /giles/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id: 940adaad860d92313516fc25c783ad0123010d7f $ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: The Giles Package 20 | # 21 | ###################################################################### 22 | 23 | """ 24 | __init__.py - package implementation 25 | """ 26 | 27 | __author__ = "Rob King" 28 | __copyright__ = "Copyright (C) 2011-2013 KoreLogic, Inc. All Rights Reserved." 29 | __credits__ = [] 30 | __license__ = "See README.LICENSE" 31 | __version__ = "$Id: 940adaad860d92313516fc25c783ad0123010d7f $" 32 | __maintainer__ = "Rob King" 33 | __email__ = "rking@korelogic.com" 34 | __status__ = "Alpha" 35 | 36 | ###################################################################### 37 | # 38 | # Version 39 | # 40 | ###################################################################### 41 | 42 | version = 0x30002800 43 | 44 | 45 | def get_release_number(): 46 | """Return the current release version.""" 47 | 48 | return version 49 | 50 | 51 | def get_release_string(): 52 | """Return the current release version.""" 53 | major = (version >> 28) & 0x0f 54 | minor = (version >> 20) & 0xff 55 | patch = (version >> 12) & 0xff 56 | state = (version >> 10) & 0x03 57 | build = version & 0x03ff 58 | if state == 0: 59 | state_string = "ds" 60 | elif state == 1: 61 | state_string = "rc" 62 | elif state == 2: 63 | state_string = "sr" 64 | elif state == 3: 65 | state_string = "xs" 66 | if state == 2 and build == 0: 67 | return '%d.%d.%d' % (major, minor, patch) 68 | else: 69 | return '%d.%d.%d.%s%d' % (major, minor, patch, state_string, build) 70 | -------------------------------------------------------------------------------- /giles/caseless_string.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id$ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: Provide a case-insensitive string class. 20 | # 21 | ###################################################################### 22 | 23 | """ 24 | This module provides the CaselessString class. A CaselessString works just like 25 | a string, but ignores case for the purposes of hashing and comparisons. The 26 | original casing of the string is preserved, meaning it still pretty-prints 27 | nicely. 28 | 29 | Some quick examples: 30 | 31 | >>> CS = CaselessString 32 | >>> S = CS("HELLO, WORLD!") 33 | >>> s = CS("hello, world!") 34 | >>> s == S 35 | True 36 | 37 | >>> d = {} 38 | >>> d[s] = "A message" 39 | >>> d[S] 40 | 'A message' 41 | 42 | >>> S 43 | CaselessString('HELLO, WORLD!') 44 | >>> S.lower() 45 | CaselessString('hello, world!') 46 | >>> S 47 | CaselessString('HELLO, WORLD!') 48 | 49 | >>> "hello, world!" == S 50 | True 51 | >>> S == "hello, world!" 52 | True 53 | 54 | >>> s + " " + S 55 | CaselessString('hello, world! HELLO, WORLD!') 56 | >>> s + " " + S == "hello, world! HELLO, WORLD!" 57 | True 58 | >>> s + " " + S == "HeLlO, WoRlD! hElLo, WoRlD!" 59 | True 60 | 61 | >>> "h" in CaselessString("Hello, world!") 62 | True 63 | >>> "H" in CaselessString("hello, world!") 64 | True 65 | >>> "W" in CaselessString("hello, world!") 66 | True 67 | 68 | >>> "B" < "a" 69 | True 70 | >>> CaselessString("B") > CaselessString("a") 71 | True 72 | >>> CaselessString("B") < "a" 73 | False 74 | >>> "a" < CaselessString("B") 75 | True 76 | 77 | CaselessString also overrides all of the various transforming str methods and 78 | causes them to return a transformed CaselessString. More interestingly, it also 79 | processes their arguments when necessary to remove casing. For example: 80 | 81 | >>> "AbCdE".strip("AB") 82 | 'bCdE' 83 | >>> CaselessString("AbCdE").strip("AB") 84 | CaselessString('CdE') 85 | 86 | >>> S = CaselessString("HELLO, THIS IS THE WORLD!") 87 | >>> S.partition(" tHIs Is thE") 88 | (CaselessString('HELLO,'), CaselessString(' tHIs Is thE'), CaselessString(' WORLD!')) 89 | 90 | >>> s = "HELLO, THIS IS THE WORLD!" 91 | >>> s.partition(" THIS IS THE") 92 | ('HELLO,', ' THIS IS THE', ' WORLD!') 93 | 94 | >>> CaselessString("hello world").split() 95 | [CaselessString('hello'), CaselessString('world')] 96 | 97 | >>> CaselessString("Hello|This|Is|The|World").rsplit("|", maxsplit=2) 98 | [CaselessString('Hello|This|Is'), CaselessString('The'), CaselessString('World')] 99 | 100 | >>> "Hello|This|Is|The|World".rsplit("|", maxsplit=2) 101 | ['Hello|This|Is', 'The', 'World'] 102 | """ 103 | 104 | __author__ = "Rob King" 105 | __copyright__ = "Copyright (C) 2013-2014 KoreLogic, Inc. All Rights Reserved." 106 | __credits__ = [] 107 | __license__ = "See README.LICENSE" 108 | __version__ = "$Id$" 109 | __maintainer__ = "Rob King" 110 | __email__ = "rking@korelogic.com" 111 | __status__ = "Alpha" 112 | 113 | import re 114 | 115 | 116 | class CaselessString(str): 117 | 118 | """ 119 | The CaselessString class implements a case-insensitive but case-preserving 120 | string that in general operates just like a normal string. 121 | """ 122 | 123 | @classmethod 124 | def _add_methods(cls): 125 | def add_delegate1(method, delegate): 126 | def func(self, *args, **kwargs): 127 | return CaselessString(delegate(self._string, *args, **kwargs)) 128 | setattr(cls, method, func) 129 | 130 | for method in ("capitalize", "casefold", "center", "encode", "expandtabs", 131 | "__format__", "format", "format_map", "join", "ljust", "lower", 132 | "lstrip", "rjust", "rstrip", "strip", "upper"): 133 | add_delegate1(method, getattr(str, method)) 134 | 135 | def add_delegate2(method, delegate): 136 | def func(self, sub, start=0, end=None): 137 | if isinstance(sub, str) and isinstance(start, int) and (end is None or isinstance(end, int)): 138 | if end is None: 139 | end = len(self._folded) 140 | return delegate(self._folded, sub.casefold(), start, end) 141 | return delegate(self._string, sub, start, end) 142 | setattr(cls, method, func) 143 | 144 | for method in ("count", "endswith", "find", "index", "rfind", "rindex", "startswith"): 145 | add_delegate2(method, getattr(str, method)) 146 | 147 | def add_delegate3(method, delegate): 148 | def func(self, chars=None): 149 | if chars is None: 150 | return CaselessString(delegate(self._string, chars)) 151 | 152 | if not isinstance(chars, str): 153 | return CaselessString(delegate(self._string, chars)) 154 | 155 | chars += "".join([c.swapcase() for c in chars]) 156 | return CaselessString(delegate(self._string, chars)) 157 | setattr(cls, method, func) 158 | 159 | for method in ("lstrip", "rstrip", "strip"): 160 | add_delegate3(method, getattr(str, method)) 161 | 162 | def __init__(self, string): 163 | self._add_methods() 164 | if isinstance(string, CaselessString): 165 | self._string = string._string 166 | self._folded = string._folded 167 | 168 | elif isinstance(string, str): 169 | self._string = str(string) 170 | self._folded = string.casefold() 171 | 172 | else: 173 | raise TypeError("string must be a str or CaselessString") 174 | 175 | def __add__(self, other): 176 | return CaselessString(self._string + other) 177 | 178 | def __contains__(self, c): 179 | if isinstance(c, str): 180 | return c.casefold() in self._folded 181 | return c in self._folded 182 | 183 | def __eq__(self, other): 184 | if isinstance(other, str): 185 | return self._folded == other.casefold() 186 | return self._string == other 187 | 188 | def __hash__(self): 189 | return hash(self._folded) 190 | 191 | def __len__(self): 192 | return len(self._string) 193 | 194 | #################################################################### 195 | # 196 | # Note that we don't use total_ordering from functools here because 197 | # we want to inherit from str, which defines those methods itself 198 | # and therefore total_ordering won't override them. 199 | # 200 | #################################################################### 201 | 202 | def __lt__(self, other): 203 | if isinstance(other, str): 204 | return self._folded < other.casefold() 205 | return self._string < other 206 | 207 | def __le__(self, other): 208 | if isinstance(other, str): 209 | return self._folded <= other.casefold() 210 | return self._string <= other 211 | 212 | def __gt__(self, other): 213 | if isinstance(other, str): 214 | return self._folded > other.casefold() 215 | return self._string > other 216 | 217 | def __ge__(self, other): 218 | if isinstance(other, str): 219 | return self._folded >= other.casefold() 220 | return self._string >= other 221 | 222 | def __mod__(self, other): 223 | return CaselessString(self._string % other) 224 | 225 | def __mul__(self, count): 226 | return CaselessString(self._string * count) 227 | 228 | def __ne__(self, other): 229 | if isinstance(other, str): 230 | return self._folded != other.casefold() 231 | return self._string != other 232 | 233 | def __radd__(self, other): 234 | return CaselessString(other + self._string) 235 | 236 | def __repr__(self): 237 | return "CaselessString('%s')" % str(self).replace("'", r"\'") 238 | 239 | def __rmod__(self, other): 240 | return CaselessString(str(other) % self._string) 241 | 242 | def __rmul__(self, other): 243 | return CaselessString(other * self._string) 244 | 245 | def __str__(self): 246 | return self._string 247 | 248 | def partition(self, sep): 249 | if not isinstance(sep, str) or len(sep) == 0: 250 | return self._string.partition(sep) 251 | 252 | index = self._folded.find(sep.casefold()) 253 | if len(self._string) != len(self._folded) or len(sep) != len(sep.casefold()): 254 | return CaselessString(self._folded[0:index]), CaselessString(sep.casefold()), \ 255 | CaselessString(self._folded[index + len(sep.casefold()):]) 256 | 257 | else: 258 | return CaselessString(self._string[0:index]), CaselessString(sep), CaselessString(self._string[index + len(sep):]) 259 | 260 | def replace(self, old, new, count=None): 261 | if not isinstance(old, str) or not isinstance(new, str) or (count is not None and not isinstance(count, int)): 262 | return CaselessString(self._string.replace(old, new, count)) 263 | 264 | if count is None: 265 | count = 0 266 | 267 | return CaselessString(re.sub(re.escape(str(old)), str(new), self._string, count, re.I)) 268 | 269 | def rpartition(self, sep): 270 | if not isinstance(sep, str) or len(sep) == 0: 271 | return self._string.partition(sep) 272 | 273 | index = self._folded.find(sep.casefold()) 274 | if len(self._string) != len(self._folded) or len(sep) != len(sep.casefold()): 275 | return CaselessString(self._folded[0:index]), CaselessString(sep.casefold()), \ 276 | CaselessString(self._folded[index + len(sep.casefold()):]) 277 | 278 | else: 279 | return CaselessString(self._string[0:index]), CaselessString(sep), CaselessString(self._string[index + len(sep):]) 280 | 281 | def rsplit(self, sep=None, maxsplit=-1): 282 | result = [CaselessString(x[::-1]) for x in re.split(re.escape(sep[::-1]) if sep is not None else r"\s+", 283 | self._string[::-1], maxsplit if maxsplit > 0 else 0, flags=re.I) if len(x) > 0] 284 | result.reverse() 285 | return result 286 | 287 | def split(self, sep=None, maxsplit=-1): 288 | return [CaselessString(x) for x in re.split(re.escape(sep) 289 | if sep is not None else r"\s+", self._string, maxsplit if maxsplit > 0 else 0, flags=re.I) if len(x) > 0] 290 | 291 | def splitlines(self, *args): 292 | return [CaselessString(x) for x in str.splitlines(self._string, *args)] 293 | 294 | def translate(self, *args): 295 | return CaselessString(self._string.translate(*args)) 296 | 297 | if __name__ == "__main__": 298 | import doctest 299 | doctest.testmod() 300 | -------------------------------------------------------------------------------- /giles/expression.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id: 9ab794bf5f885e816d25fd2cfea01342ed14cc84 $ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: Parse expressions for Giles predicates and productions. 20 | # 21 | ###################################################################### 22 | 23 | """ 24 | expression.py - Parse expressions for Giles predicates and productions. 25 | """ 26 | 27 | __author__ = "Rob King" 28 | __copyright__ = "Copyright (C) 2011-2014 KoreLogic, Inc. All Rights Reserved." 29 | __credits__ = [] 30 | __license__ = "See README.LICENSE" 31 | __version__ = "$Id: 9ab794bf5f885e816d25fd2cfea01342ed14cc84 $" 32 | __maintainer__ = "Rob King" 33 | __email__ = "rking@korelogic.com" 34 | __status__ = "Alpha" 35 | 36 | import re 37 | 38 | from giles import pyre 39 | from giles.caseless_string import CaselessString as CS 40 | 41 | ###################################################################### 42 | # 43 | # The DefaultOperationsMixin is a mixin class that provides all of the 44 | # basic arithmetic/logic operations via the special method names 45 | # defined by the Python data model. 46 | # 47 | # The basic theory of operation here is that upon evaluation of an 48 | # expression, the parser applies normal Python operators for the 49 | # various operations (+, -, /, etc). For numbers and strings and 50 | # such, this just works. For Giles datatypes, these methods are 51 | # invoked and produce combining nodes which end up building up the 52 | # syntax tree (and folding constants automatically). 53 | # 54 | # It's actually pretty elegant when you think about it. 55 | # 56 | ###################################################################### 57 | 58 | 59 | class DefaultOperationsMixin: 60 | 61 | def __lt__(self, other): 62 | return BinaryOpNode("<", self, other, bool) 63 | 64 | def __le__(self, other): 65 | return BinaryOpNode("<=", self, other, bool) 66 | 67 | def __eq__(self, other): 68 | return BinaryOpNode("=", self, other, bool) 69 | 70 | def __ne__(self, other): 71 | return BinaryOpNode("!=", self, other, bool) 72 | 73 | def __gt__(self, other): 74 | return BinaryOpNode(">", self, other, bool) 75 | 76 | def __ge__(self, other): 77 | return BinaryOpNode(">=", self, other, bool) 78 | 79 | def __add__(self, other): 80 | return BinaryOpNode("+", self, other) 81 | 82 | def __sub__(self, other): 83 | return BinaryOpNode("-", self, other) 84 | 85 | def __mul__(self, other): 86 | return BinaryOpNode("*", self, other) 87 | 88 | def __truediv__(self, other): 89 | return BinaryOpNode("/", self, other) 90 | 91 | def __mod__(self, other): 92 | return BinaryOpNode("%", self, other) 93 | 94 | def __radd__(self, other): 95 | return BinaryOpNode("+", other, self) 96 | 97 | def __rsub__(self, other): 98 | return BinaryOpNode("-", other, self) 99 | 100 | def __rmul__(self, other): 101 | return BinaryOpNode("*", other, self) 102 | 103 | def __rtruediv__(self, other): 104 | return BinaryOpNode("/", other, self) 105 | 106 | def __rmod__(self, other): 107 | return BinaryOpNode("%", other, self) 108 | 109 | def __neg__(self): 110 | return UnaryOpNode("neg", self) 111 | 112 | def __pos__(self): 113 | return self 114 | 115 | ###################################################################### 116 | # 117 | # The Node class is the basic class of the abstract syntax tree 118 | # generated for an expression - expressions are trees whose internal 119 | # nodes are (subclasses of) Nodes and whose leaves are either Nodes or 120 | # normal Python constants. 121 | # 122 | # Nodes carry a type with them in their "type" member. What's 123 | # important to understand is that all nodes have a type, even if 124 | # they're abstract - this information is used by the type-checker. 125 | # 126 | # It's important to understand that Nodes are only built if one or 127 | # more of their consituent parts can't be entirely determined at 128 | # compile-time. In other words, having even a single Node by 129 | # definition means that an expression is not constant. 130 | # 131 | ###################################################################### 132 | 133 | 134 | class Node: 135 | 136 | def __init__(self): 137 | self.type = None 138 | 139 | def __repr__(self): 140 | return str(self) 141 | 142 | ###################################################################### 143 | # 144 | # A LocalReferenceNode represents a reference to a local variable in 145 | # an expression that will be resolved at runtime. 146 | # 147 | ###################################################################### 148 | 149 | 150 | class LocalReferenceNode(Node, DefaultOperationsMixin): 151 | 152 | def __init__(self, variable, type): 153 | self.type = type 154 | self.variable = variable 155 | 156 | def __str__(self): 157 | return "Locals.%s" % self.variable 158 | 159 | ###################################################################### 160 | # 161 | # A ThisReferenceNode represents a reference to a field in the event 162 | # currently being matched. 163 | # 164 | ###################################################################### 165 | 166 | 167 | class ThisReferenceNode(Node, DefaultOperationsMixin): 168 | 169 | def __init__(self, variable, type): 170 | self.type = type 171 | self.variable = variable 172 | 173 | def __str__(self): 174 | return "This.%s" % self.variable 175 | 176 | ###################################################################### 177 | # 178 | # A BinaryOpNode represents a binary operator and its operands. 179 | # It can either have a passed-in type, or it will take the type of 180 | # its arguments (this works because Giles's type system does not do 181 | # implicit type casting or promotion, so both operands will always be 182 | # of the same type). We allow a passed-in type because, for example, 183 | # the comparison operators take in arbitrary types but return bools. 184 | # 185 | ###################################################################### 186 | 187 | 188 | class BinaryOpNode(Node, DefaultOperationsMixin): 189 | 190 | def __init__(self, operation, arg1, arg2, kind=None, readable_name=None): 191 | self.operation = operation 192 | self.name = readable_name if readable_name is not None else operation 193 | self.arg1 = arg1 194 | self.arg2 = arg2 195 | self.type = type(arg1) if type(arg1) in (bool, float, int, str) else arg1.type 196 | 197 | if kind is not None: 198 | self.type = kind 199 | 200 | def __str__(self): 201 | arg1 = self.arg1 202 | arg2 = self.arg2 203 | if isinstance(self.arg1, str): 204 | arg1 = "'%s'" % arg1.replace("'", "''") 205 | 206 | if isinstance(self.arg2, str): 207 | arg2 = "'%s'" % arg2.replace("'", "''") 208 | 209 | return "(%s %s %s)" % (arg1, self.name, arg2) 210 | 211 | ###################################################################### 212 | # 213 | # A UnaryOpNode represents a unary operator and its operand. 214 | # 215 | ###################################################################### 216 | 217 | 218 | class UnaryOpNode(Node, DefaultOperationsMixin): 219 | 220 | def __init__(self, operation, arg1, readable_name=None): 221 | self.operation = operation 222 | self.name = readable_name if readable_name is not None else operation 223 | self.arg1 = arg1 224 | self.type = type(arg1) if type(arg1) in (bool, float, int, str) else arg1.type 225 | 226 | def __str__(self): 227 | arg1 = self.arg1 228 | if isinstance(self.arg1, str): 229 | arg1 = "'%s'" % arg1.replace("'", "''") 230 | 231 | return "(%s %s)" % (self.name, arg1) 232 | 233 | ###################################################################### 234 | # 235 | # An IfNode represents the ternary-if operation. This is produced by 236 | # the "if" function in an expression. 237 | # 238 | ###################################################################### 239 | 240 | 241 | class IfNode(Node, DefaultOperationsMixin): 242 | 243 | def __init__(self, predicate, if_true, if_false): 244 | self.predicate = predicate 245 | self.if_true = if_true 246 | self.if_false = if_false 247 | self.type = type(if_true) if type(if_true) in (bool, float, int, str) else if_true.type 248 | 249 | def __str__(self): 250 | return "if(%s, %s, %s)" % (self.predicate, self.if_true, self.if_false) 251 | 252 | ###################################################################### 253 | # 254 | # A CastNode represents a cast of a value to a different (or, I 255 | # suppose, the same) data type. 256 | # 257 | ###################################################################### 258 | 259 | 260 | class CastNode(Node, DefaultOperationsMixin): 261 | 262 | def __init__(self, expression, cast_to): 263 | self.expression = expression 264 | self.type = cast_to 265 | 266 | def __str__(self): 267 | return "cast(%s -> %s)" % (self.expression, self.type) 268 | 269 | ###################################################################### 270 | # 271 | # A JoinNode represents a conjunction of two boolean predicates on 272 | # variables. JoinNodes represent joins in the runtime production 273 | # system - so they're not just normal boolean conjunctions. 274 | # 275 | # Both children of a JoinNode must be boolean expressions, and the 276 | # left-hand operand of each expression must be a single variable. 277 | # The JoinNode is then used at runtime to produce the join statement 278 | # between those two variables. 279 | # 280 | ###################################################################### 281 | 282 | 283 | class JoinNode(Node, DefaultOperationsMixin): 284 | 285 | def __init__(self, arg1, arg2): 286 | self.left = arg1 287 | self.right = arg2 288 | self.type = bool 289 | 290 | def __str__(self): 291 | return "%s AND %s" % (self.left, self.right) 292 | 293 | ###################################################################### 294 | # 295 | # A FunctionNode represents an invocation of a SQL function inside 296 | # an expression. 297 | # 298 | ###################################################################### 299 | 300 | 301 | class FunctionNode(Node, DefaultOperationsMixin): 302 | 303 | def __init__(self, name, external, returns, args): 304 | self.name = name 305 | self.external = external 306 | self.type = returns 307 | self.args = list(args) 308 | 309 | def __str__(self): 310 | return "%s%s" % (self.name, self.args) 311 | 312 | ###################################################################### 313 | # 314 | # The Tokenizer is an instance of a normal Pyre tokenizer, with 315 | # definitions for all of the tokens in the Giles language. 316 | # 317 | # The Tokenizer performs variable resolution at tokenization time, 318 | # which allows for more aggressive constant folding and a cleaner code 319 | # path later. 320 | # 321 | # This subclass of the Tokenizer object is not reentrant - you need 322 | # to create a new Tokenizer object for every expression. 323 | # 324 | ###################################################################### 325 | 326 | 327 | class Tokenizer(pyre.Tokenizer): 328 | 329 | def __init__(self, constants=None, variables=None, this=None): 330 | """ 331 | Create a new Tokenizer. 332 | 333 | constants - the currently defined constants 334 | variables - the current local rule scope 335 | this - the current matched event 336 | """ 337 | 338 | super().__init__() 339 | 340 | self.constants = constants if constants is not None else {} 341 | self.variables = variables if variables is not None else {} 342 | self.this = this if this is not None else {} 343 | 344 | @pyre.syntax_rule(r"(!?~|[=!<>]=?|&&|[|][|]|[-+*/%.])", flags=re.I) 345 | def make_operator(self, tokenizer, captures): 346 | return pyre.OperatorToken(CS(captures[0])) 347 | 348 | @pyre.syntax_rule(r"Constants[.]([A-Za-z][A-Za-z0-9]*)", flags=re.I) 349 | def make_constant_reference(self, tokenizer, captures): 350 | if CS(captures[1]) not in self.constants: 351 | raise Exception("Unknown constant: '%s'" % captures[0]) 352 | 353 | return self.constants[CS(captures[1])] 354 | 355 | @pyre.syntax_rule(r"This[.]([A-Za-z][A-Za-z0-9]*)", flags=re.I) 356 | def make_this_reference(self, tokenizer, captures): 357 | if CS(captures[1]) not in tokenizer.this: 358 | raise Exception("Unknown field: '%s'" % captures[0]) 359 | 360 | return ThisReferenceNode(CS(captures[1]), tokenizer.this[CS(captures[1])]) 361 | 362 | @pyre.syntax_rule(r"Locals[.]([A-Za-z][A-Za-z0-9]*)", flags=re.I) 363 | def make_locals_reference(self, tokenizer, captures): 364 | if CS(captures[1]) not in tokenizer.variables: 365 | raise Exception("Unknown variable: '%s'" % captures[0]) 366 | 367 | return LocalReferenceNode(CS(captures[1]), tokenizer.variables[CS(captures[1])]) 368 | 369 | @pyre.syntax_rule(r"[a-z][a-z0-9_]*", flags=re.I) 370 | def make_function(self, tokenizer, captures): 371 | if captures[0].lower() in ("and", "not", "like", "unlike"): 372 | return pyre.OperatorToken(CS(captures[0])) 373 | 374 | return pyre.FunctionToken(CS(captures[0])) 375 | 376 | @pyre.syntax_rule(r"(true|false)", flags=re.I) 377 | def make_boolean(self, tokenizer, captures): 378 | return captures[0].lower() == "true" 379 | 380 | @pyre.syntax_rule(r"\d+[.]\d+(e[-]?\d+)?", flags=re.I) 381 | def make_real(self, tokenizer, captures): 382 | return float(captures[0]) 383 | 384 | @pyre.syntax_rule(r"\d+") 385 | def make_int(self, tokenizer, captures): 386 | return int(captures[0]) 387 | 388 | @pyre.syntax_rule(r"'([^']*)'") 389 | def make_single_quoted_string(self, tokenizer, captures): 390 | return captures[1] 391 | 392 | @pyre.syntax_rule(r'"([^"]*)"') 393 | def make_double_quoted_string(self, tokenizer, captures): 394 | return captures[1] 395 | 396 | @pyre.syntax_rule(r"[$][a-f0-9]{2}", flags=re.I) 397 | def make_character_reference(self, tokenizer, captures): 398 | return chr(int(captures[0][1:], 16)) 399 | 400 | @pyre.syntax_rule(r"#[^\n]*", skip=True) 401 | def make_comment(self, tokenizer, captures): 402 | pass 403 | 404 | ###################################################################### 405 | # 406 | # The type_check function is a method decorator that can be applied to 407 | # parser rules. The decorator's arguments are tuples, and each tuple 408 | # represents a valid combination of arguments. For example, the 409 | # decoration of the "+" operator might look like this: 410 | # 411 | # @type_check((float, float), (int, int)) 412 | # 413 | # Meaning that it takes two floats or two ints as an argument. 414 | # 415 | # For single-operand or single-argument functions, a single type 416 | # argument may be passed. 417 | # 418 | ###################################################################### 419 | 420 | 421 | def type_check(*combinations): 422 | def wrap(func): 423 | def inner(self, parser, *args): 424 | name = None 425 | if hasattr(func, 'binary_operator_name'): 426 | name = "operator '%s'" % func.binary_operator_name 427 | 428 | elif hasattr(func, 'unary_operator_name'): 429 | name = "operator '%s'" % func.unary_operator_name 430 | 431 | elif hasattr(func, 'defined_function_name'): 432 | name = "function '%s'" % func.defined_function_name 433 | 434 | else: 435 | assert False 436 | 437 | types = tuple([arg.type if isinstance(arg, Node) else type(arg) for arg in args]) 438 | if len(types) == 1: 439 | types = types[0] 440 | 441 | if types not in combinations: 442 | if type(types) is not list: 443 | types = [types] 444 | raise Exception("Invalid types for function/operator '%s': %s" % (name, " and ".join([str(x) for x in types]))) 445 | 446 | return func(self, parser, *args) 447 | 448 | for attribute in ('binary_operator_name', 'binary_operator_precedence', 'binary_operator_associativity', 449 | 'unary_operator_name', 'unary_operator_precedence', 'unary_operator_associativity', 450 | 'defined_function_name'): 451 | if hasattr(func, attribute): 452 | setattr(inner, attribute, getattr(func, attribute)) 453 | 454 | return inner 455 | return wrap 456 | 457 | ###################################################################### 458 | # 459 | # Here's where the magic happens. This parser (a normal Pyre parser) 460 | # defines all of the semantics of the Giles expression language. 461 | # Expressions are passed into the parser and it builds up abstract 462 | # syntax trees which are returned to the compiler. 463 | # 464 | # See the comments above to see how the parser builds the abstract 465 | # syntax tree as a side-effect of evaluating the expression. 466 | # 467 | ###################################################################### 468 | 469 | 470 | class Parser(pyre.Parser): 471 | 472 | def __init__(self, constants=None, variables=None, this=None, allow_regexp=False): 473 | """ 474 | Create a new parser. 475 | 476 | constants - the currently defined constants 477 | variables - the current local rule scope 478 | this - the current matched event 479 | allow_regexp - allow regular expression operations 480 | """ 481 | 482 | super().__init__() 483 | 484 | self.constants = constants if constants is not None else {} 485 | self.variables = variables if variables is not None else {} 486 | self.this = this if this is not None else {} 487 | self.allow_regexp = allow_regexp 488 | 489 | @type_check((float, float), (int, int)) 490 | @pyre.binary_operator("+", precedence=10) 491 | def operator_add(self, parser, arg1, arg2): 492 | return arg1 + arg2 493 | 494 | @type_check((float, float), (int, int)) 495 | @pyre.binary_operator("-", precedence=10) 496 | def operator_sub(self, parser, arg1, arg2): 497 | return arg1 - arg2 498 | 499 | @type_check((float, float), (int, int)) 500 | @pyre.binary_operator("*", precedence=20) 501 | def operator_mul(self, parser, arg1, arg2): 502 | return arg1 * arg2 503 | 504 | @type_check((float, float), (int, int)) 505 | @pyre.binary_operator("/", precedence=20) 506 | def operator_div(self, parser, arg1, arg2): 507 | return arg1 / arg2 508 | 509 | @type_check((int, int)) 510 | @pyre.binary_operator("%", precedence=20) 511 | def operator_mod(self, parser, arg1, arg2): 512 | return arg1 % arg2 513 | 514 | @type_check((bool, bool), (float, float), (int, int), (str, str)) 515 | @pyre.binary_operator("==", precedence=5) 516 | def operator_equ(self, parser, arg1, arg2): 517 | return arg1 == arg2 518 | 519 | @type_check((bool, bool), (float, float), (int, int), (str, str)) 520 | @pyre.binary_operator("<=", precedence=5) 521 | def operator_lte(self, parser, arg1, arg2): 522 | return arg1 <= arg2 523 | 524 | @type_check((bool, bool), (float, float), (int, int), (str, str)) 525 | @pyre.binary_operator(">=", precedence=5) 526 | def operator_gte(self, parser, arg1, arg2): 527 | return arg1 >= arg2 528 | 529 | @type_check((bool, bool), (float, float), (int, int), (str, str)) 530 | @pyre.binary_operator("!=", precedence=5) 531 | def operator_neq(self, parser, arg1, arg2): 532 | return arg1 != arg2 533 | 534 | @type_check((bool, bool), (float, float), (int, int), (str, str)) 535 | @pyre.binary_operator("<", precedence=5) 536 | def operator_lt(self, parser, arg1, arg2): 537 | return arg1 < arg2 538 | 539 | @type_check((bool, bool), (float, float), (int, int), (str, str)) 540 | @pyre.binary_operator(">", precedence=5) 541 | def operator_gt(self, parser, arg1, arg2): 542 | return arg1 > arg2 543 | 544 | @type_check((bool, bool)) 545 | @pyre.binary_operator("||", precedence=30) 546 | def operator_or(self, parser, arg1, arg2): 547 | if type(arg1) == bool and type(arg2) == bool: 548 | return arg1 or arg2 549 | 550 | return BinaryOpNode("OR", arg1, arg2) 551 | 552 | @type_check((bool, bool)) 553 | @pyre.binary_operator("&&", precedence=40) 554 | def operator_and(self, parser, arg1, arg2): 555 | if type(arg1) == bool and type(arg2) == bool: 556 | return arg1 and arg2 557 | 558 | return BinaryOpNode("AND", arg1, arg2) 559 | 560 | @type_check((str, str)) 561 | @pyre.binary_operator("~", precedence=5) 562 | def operator_regexp(self, parser, arg1, arg2): 563 | if not self.allow_regexp: 564 | raise Exception("regular expressions in expressions are disabled") 565 | 566 | if type(arg2) == str: 567 | try: 568 | re.compile(arg2) 569 | 570 | except Exception as e: 571 | raise Exception("Invalid regular expression: '%s'" % e) 572 | 573 | if type(arg1) == str and type(arg2) == str: 574 | try: 575 | return re.match(arg2, arg1) is not None 576 | 577 | except Exception as e: 578 | raise Exception("Invalid regular expression: '%s'" % e) 579 | 580 | else: 581 | return BinaryOpNode("REGEXP", arg1, arg2, bool) 582 | 583 | @type_check((str, str)) 584 | @pyre.binary_operator("!~", precedence=5) 585 | def operator_notregexp(self, parser, arg1, arg2): 586 | if not self.allow_regexp: 587 | raise Exception("regular expressions in expressions are disabled") 588 | 589 | if type(arg2) == str: 590 | try: 591 | re.compile(arg2) 592 | 593 | except Exception as e: 594 | raise Exception("Invalid regular expression: '%s'" % e) 595 | 596 | if type(arg1) == str and type(arg2) == str: 597 | try: 598 | return re.match(arg2, arg1) is None 599 | 600 | except Exception as e: 601 | raise Exception("Invalid regular expression: '%s'" % e) 602 | 603 | else: 604 | return BinaryOpNode("NOT REGEXP", arg1, arg2, bool) 605 | 606 | @type_check((str, str)) 607 | @pyre.binary_operator("like", precedence=5) 608 | def operator_like(self, parser, arg1, arg2): 609 | if type(arg1) == str and type(arg2) == str: 610 | pattern = re.escape(arg2) 611 | pattern = pattern.replace(r"\%", "%") 612 | pattern = pattern.replace(r"\_", "_") 613 | pattern = pattern.replace("%", ".*") 614 | pattern = pattern.replace("_", ".") 615 | 616 | return re.match("^" + pattern + "$", arg1) is not None 617 | 618 | else: 619 | return BinaryOpNode("LIKE", arg1, arg2, bool) 620 | 621 | @type_check((str, str)) 622 | @pyre.binary_operator("unlike", precedence=5) 623 | def operator_unlike(self, parser, arg1, arg2): 624 | if type(arg1) == str and type(arg2) == str: 625 | pattern = re.escape(arg2) 626 | pattern = pattern.replace(r"\%", "%") 627 | pattern = pattern.replace(r"\_", "_") 628 | pattern = pattern.replace("%", ".*") 629 | pattern = pattern.replace("_", ".") 630 | 631 | return re.match("^" + pattern + "$", arg1) is None 632 | 633 | else: 634 | return BinaryOpNode("NOT LIKE", arg1, arg2, bool) 635 | 636 | @type_check((bool, bool)) 637 | @pyre.binary_operator("and", precedence=0) 638 | def operator_join(self, parser, arg1, arg2): 639 | if parser.this is None: 640 | raise Exception("Logical conjunctions of conditions are valid only in match predicates") 641 | 642 | return JoinNode(arg1, arg2) 643 | 644 | @type_check((str, str)) 645 | @pyre.binary_operator(".", precedence=20) 646 | def operator_concat(self, parser, arg1, arg2): 647 | if type(arg1) == str and type(arg2) == str: 648 | return arg1 + arg2 649 | 650 | return BinaryOpNode("||", arg1, arg2, readable_name=".") 651 | 652 | @type_check(float, int) 653 | @pyre.unary_operator("+", precedence=98) 654 | def operator_pos(self, parser, arg1): 655 | return +arg1 656 | 657 | @type_check(float, int) 658 | @pyre.unary_operator("-", precedence=98) 659 | def operator_neg(self, parser, arg1): 660 | return -arg1 661 | 662 | @type_check(bool) 663 | @pyre.unary_operator("not", precedence=98) 664 | def operator_not(self, parser, arg1): 665 | if type(arg1) == bool: 666 | return not arg1 667 | 668 | return UnaryOpNode("NOT", arg1) 669 | 670 | @type_check(bool) 671 | @pyre.defined_function("string_of_bool") 672 | def function_string_of_bool(self, parser, arg1): 673 | return str(arg1) if type(arg1) == bool else CastNode(arg1, str) 674 | 675 | @type_check(float) 676 | @pyre.defined_function("string_of_real") 677 | def function_string_of_real(self, parser, arg1): 678 | return str(arg1) if type(arg1) == float else CastNode(arg1, str) 679 | 680 | @type_check(int) 681 | @pyre.defined_function("string_of_int") 682 | def function_string_of_int(self, parser, arg1): 683 | return str(arg1) if type(arg1) == int else CastNode(arg1, str) 684 | 685 | @type_check(int) 686 | @pyre.defined_function("real_of_int") 687 | def function_real_of_int(self, parser, arg1): 688 | return float(arg1) if type(arg1) == int else CastNode(arg1, float) 689 | 690 | @type_check(float) 691 | @pyre.defined_function("int_of_real") 692 | def function_int_of_real(self, parser, arg1): 693 | return int(arg1) if type(arg1) == float else CastNode(arg1, int) 694 | 695 | @type_check(str) 696 | @pyre.defined_function("int_of_string") 697 | def function_int_of_string(self, parser, arg1): 698 | return int(arg1) if type(arg1) == str else CastNode(arg1, int) 699 | 700 | @type_check((bool, bool, bool), (bool, float, float), (bool, int, int), (bool, str, str)) 701 | @pyre.defined_function("if") 702 | def function_if(self, parser, predicate, if_true, if_false): 703 | if type(predicate) == bool: 704 | return if_true if predicate else if_false 705 | 706 | return IfNode(predicate, if_true, if_false) 707 | 708 | ###################################################################### 709 | # 710 | # DelayedExpressions are expressions that are read in from the 711 | # input file but not yet evaluated. Evaluation is delayed because 712 | # expressions might contain references to things that haven't yet been 713 | # defined (constants, etc). 714 | # 715 | # Expressions are marked with the tag "!expr" in the rule file. 716 | # 717 | ###################################################################### 718 | 719 | 720 | class DelayedExpression: 721 | 722 | """An expression to be evaluated.""" 723 | 724 | def __init__(self, expression): 725 | if not isinstance(expression, str): 726 | raise Exception("Invalid expression: not a string") 727 | 728 | self.expression = expression 729 | 730 | def __repr__(self): 731 | return str(self) 732 | 733 | def __str__(self): 734 | return self.expression 735 | -------------------------------------------------------------------------------- /giles/forbidden_names.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id: be8f2653eef802fff293bbada8b57658a1c79f68 $ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: List names that cannot be used as variable or field names 20 | # due to conflict with keywords in SQL dialects. 21 | # 22 | ###################################################################### 23 | 24 | """ 25 | This module provides a reguar expression that matches names that are invalid 26 | Giles names. This list is long, but we want to be sure we don't conflict with 27 | any popular SQL keywords. 28 | 29 | Really, this module just: 30 | 31 | >>> import re 32 | >>> re.compile("(" + "|".join(names) + ")") is not None 33 | True 34 | """ 35 | 36 | __author__ = "Rob King" 37 | __copyright__ = "Copyright (C) 2011-2014 KoreLogic, Inc. All Rights Reserved." 38 | __credits__ = [] 39 | __license__ = "See README.LICENSE" 40 | __version__ = "$Id: be8f2653eef802fff293bbada8b57658a1c79f68 $" 41 | __maintainer__ = "Rob King" 42 | __email__ = "rking@korelogic.com" 43 | __status__ = "Alpha" 44 | 45 | names = set([ 46 | "_id", 47 | "_rowid", 48 | "a", 49 | "abort", 50 | "abs", 51 | "absolute", 52 | "access", 53 | "action", 54 | "ada", 55 | "add", 56 | "admin", 57 | "after", 58 | "aggregate", 59 | "alias", 60 | "all", 61 | "allocate", 62 | "also", 63 | "alter", 64 | "always", 65 | "analyse", 66 | "analyze", 67 | "and", 68 | "any", 69 | "are", 70 | "array", 71 | "as", 72 | "asc", 73 | "asensitive", 74 | "assertion", 75 | "assignment", 76 | "asymmetric", 77 | "at", 78 | "atomic", 79 | "attribute", 80 | "attributes", 81 | "audit", 82 | "authorization", 83 | "auto_increment", 84 | "av", 85 | "avg_row_length", 86 | "backup", 87 | "backward", 88 | "before", 89 | "begin", 90 | "bernoulli", 91 | "between", 92 | "bigint", 93 | "binary", 94 | "bit", 95 | "bit_length", 96 | "bitvar", 97 | "blob", 98 | "bool", 99 | "boolean", 100 | "both", 101 | "breadth", 102 | "break", 103 | "browse", 104 | "bulk", 105 | "by", 106 | "c", 107 | "cache", 108 | "call", 109 | "called", 110 | "cardinality", 111 | "cascade", 112 | "cascaded", 113 | "case", 114 | "cast", 115 | "catalo", 116 | "catalog_name", 117 | "ceil", 118 | "ceilin", 119 | "chain", 120 | "change", 121 | "char", 122 | "char_length", 123 | "character", 124 | "character_length", 125 | "character_set_catalo", 126 | "character_set_name", 127 | "character_set_schema", 128 | "characteristics", 129 | "characters", 130 | "check", 131 | "checked", 132 | "checkpoint", 133 | "checksum", 134 | "class", 135 | "class_origin", 136 | "clob", 137 | "close", 138 | "cluster", 139 | "clustered", 140 | "coalesce", 141 | "cobol", 142 | "collate", 143 | "collation", 144 | "collation_catalo", 145 | "collation_name", 146 | "collation_schema", 147 | "collect", 148 | "column", 149 | "column_name", 150 | "columns", 151 | "command_function", 152 | "command_function_code", 153 | "comment", 154 | "commit", 155 | "committed", 156 | "completion", 157 | "compress", 158 | "compute", 159 | "condition", 160 | "condition_number", 161 | "connect", 162 | "connection", 163 | "connection_name", 164 | "constant", 165 | "constants", 166 | "constraint", 167 | "constraint_catalo", 168 | "constraint_name", 169 | "constraint_schema", 170 | "constraints", 171 | "constructor", 172 | "contains", 173 | "containstable", 174 | "continue", 175 | "conversion", 176 | "convert", 177 | "copy", 178 | "corr", 179 | "correspondin", 180 | "count", 181 | "covar_pop", 182 | "covar_samp", 183 | "create", 184 | "createdb", 185 | "createrole", 186 | "createuser", 187 | "cross", 188 | "csv", 189 | "cube", 190 | "cume_dist", 191 | "current", 192 | "current_date", 193 | "current_default_transform_group", 194 | "current_path", 195 | "current_role", 196 | "current_time", 197 | "current_timestamp", 198 | "current_transform_group_for_type", 199 | "current_user", 200 | "cursor", 201 | "cursor_name", 202 | "cycle", 203 | "data", 204 | "database", 205 | "databases", 206 | "date", 207 | "datetime", 208 | "datetime_interval_code", 209 | "datetime_interval_precision", 210 | "day", 211 | "day_hour", 212 | "day_microsecond", 213 | "day_minute", 214 | "day_second", 215 | "dayofmonth", 216 | "dayofweek", 217 | "dayofyear", 218 | "dbcc", 219 | "deallocate", 220 | "dec", 221 | "decimal", 222 | "declare", 223 | "default", 224 | "defaults", 225 | "deferrable", 226 | "deferred", 227 | "defined", 228 | "definer", 229 | "degree", 230 | "delay_key_write", 231 | "delayed", 232 | "delete", 233 | "delimiter", 234 | "delimiters", 235 | "dense_rank", 236 | "deny", 237 | "depth", 238 | "deref", 239 | "derived", 240 | "desc", 241 | "describe", 242 | "descriptor", 243 | "destroy", 244 | "destructor", 245 | "deterministic", 246 | "diagnostics", 247 | "dictionary", 248 | "disable", 249 | "disconnect", 250 | "disk", 251 | "dispatch", 252 | "distinct", 253 | "distinctrow", 254 | "distributed", 255 | "div", 256 | "do", 257 | "domain", 258 | "double", 259 | "drop", 260 | "dual", 261 | "dummy", 262 | "dump", 263 | "dynamic", 264 | "dynamic_function", 265 | "dynamic_function_code", 266 | "each", 267 | "element", 268 | "else", 269 | "elseif", 270 | "enable", 271 | "enclosed", 272 | "encodin", 273 | "encrypted", 274 | "end", 275 | "end-exec", 276 | "enum", 277 | "equals", 278 | "errlvl", 279 | "escape", 280 | "escaped", 281 | "event", 282 | "every", 283 | "except", 284 | "exception", 285 | "exclude", 286 | "excludin", 287 | "exclusive", 288 | "exec", 289 | "execute", 290 | "existin", 291 | "exists", 292 | "exit", 293 | "exp", 294 | "explain", 295 | "external", 296 | "extract", 297 | "false", 298 | "fetch", 299 | "fields", 300 | "file", 301 | "fillfactor", 302 | "filter", 303 | "final", 304 | "first", 305 | "float", 306 | "float4", 307 | "float8", 308 | "floor", 309 | "flush", 310 | "followin", 311 | "for", 312 | "force", 313 | "foreign", 314 | "fortran", 315 | "forward", 316 | "found", 317 | "frame", 318 | "free", 319 | "freetext", 320 | "freetexttable", 321 | "freeze", 322 | "from", 323 | "full", 324 | "fulltext", 325 | "function", 326 | "fusion", 327 | "", 328 | "general", 329 | "generated", 330 | "get", 331 | "global", 332 | "go", 333 | "goto", 334 | "grant", 335 | "granted", 336 | "grants", 337 | "greatest", 338 | "group", 339 | "groupin", 340 | "handler", 341 | "havin", 342 | "header", 343 | "heap", 344 | "hierarchy", 345 | "high_priority", 346 | "hold", 347 | "holdlock", 348 | "host", 349 | "hosts", 350 | "hour", 351 | "hour_microsecond", 352 | "hour_minute", 353 | "hour_second", 354 | "id", 355 | "identified", 356 | "identity", 357 | "identity_insert", 358 | "identitycol", 359 | "if", 360 | "ignore", 361 | "ilike", 362 | "immediate", 363 | "immutable", 364 | "implementation", 365 | "implicit", 366 | "in", 367 | "include", 368 | "includin", 369 | "increment", 370 | "index", 371 | "indicator", 372 | "infile", 373 | "infix", 374 | "inherit", 375 | "inherits", 376 | "initial", 377 | "initialize", 378 | "initially", 379 | "inner", 380 | "inout", 381 | "input", 382 | "insensitive", 383 | "insert", 384 | "insert_id", 385 | "instance", 386 | "instantiable", 387 | "instead", 388 | "int", 389 | "int1", 390 | "int2", 391 | "int3", 392 | "int4", 393 | "int8", 394 | "integer", 395 | "intersect", 396 | "intersection", 397 | "interval", 398 | "into", 399 | "invoker", 400 | "is", 401 | "isam", 402 | "isnull", 403 | "isolation", 404 | "iterate", 405 | "join", 406 | "k", 407 | "key", 408 | "key_member", 409 | "key_type", 410 | "keys", 411 | "kill", 412 | "lancompiler", 413 | "language", 414 | "large", 415 | "last", 416 | "last_insert_id", 417 | "lateral", 418 | "leadin", 419 | "least", 420 | "leave", 421 | "left", 422 | "length", 423 | "less", 424 | "level", 425 | "like", 426 | "limit", 427 | "lineno", 428 | "lines", 429 | "listen", 430 | "ln", 431 | "load", 432 | "local", 433 | "locals", 434 | "localtime", 435 | "localtimestamp", 436 | "location", 437 | "locator", 438 | "lock", 439 | "login", 440 | "logs", 441 | "lon", 442 | "longblob", 443 | "longtext", 444 | "loop", 445 | "low_priority", 446 | "lower", 447 | "m", 448 | "map", 449 | "match", 450 | "matched", 451 | "max", 452 | "max_rows", 453 | "maxextents", 454 | "maxvalue", 455 | "meanin", 456 | "mediumblob", 457 | "mediumint", 458 | "mediumtext", 459 | "member", 460 | "merge", 461 | "message_length", 462 | "message_octet_length", 463 | "message_text", 464 | "method", 465 | "middleint", 466 | "min", 467 | "min_rows", 468 | "minus", 469 | "minute", 470 | "minute_microsecond", 471 | "minute_second", 472 | "minvalue", 473 | "mlslabel", 474 | "mod", 475 | "mode", 476 | "modifies", 477 | "modify", 478 | "module", 479 | "month", 480 | "monthname", 481 | "more", 482 | "move", 483 | "multiset", 484 | "mumps", 485 | "myisam", 486 | "national", 487 | "natural", 488 | "nchar", 489 | "nclob", 490 | "nestin", 491 | "new", 492 | "next", 493 | "no", 494 | "no_write_to_binlo", 495 | "noaudit", 496 | "nocheck", 497 | "nocompress", 498 | "nocreatedb", 499 | "nocreaterole", 500 | "nocreateuser", 501 | "noinherit", 502 | "nologin", 503 | "nonclustered", 504 | "none", 505 | "normalize", 506 | "normalized", 507 | "nosuperuser", 508 | "not", 509 | "nothin", 510 | "notify", 511 | "notnull", 512 | "nowait", 513 | "null", 514 | "nullable", 515 | "nullif", 516 | "nulls", 517 | "number", 518 | "numeric", 519 | "object", 520 | "octet_length", 521 | "octets", 522 | "of", 523 | "off", 524 | "offline", 525 | "offset", 526 | "offsets", 527 | "oids", 528 | "old", 529 | "on", 530 | "online", 531 | "only", 532 | "open", 533 | "opendatasource", 534 | "openquery", 535 | "openrowset", 536 | "openxml", 537 | "operation", 538 | "operator", 539 | "optimize", 540 | "option", 541 | "optionally", 542 | "options", 543 | "or", 544 | "order", 545 | "orderin", 546 | "ordinality", 547 | "others", 548 | "out", 549 | "outer", 550 | "outfile", 551 | "output", 552 | "over", 553 | "overlaps", 554 | "overlay", 555 | "overridin", 556 | "owner", 557 | "pack_keys", 558 | "pad", 559 | "parameter", 560 | "parameters", 561 | "parameter_mode", 562 | "parameter_name", 563 | "parameter_ordinal_position", 564 | "parameter_specific_catalo", 565 | "parameter_specific_name", 566 | "parameter_specific_schema", 567 | "parameters", 568 | "partial", 569 | "partition", 570 | "pascal", 571 | "password", 572 | "path", 573 | "pctfree", 574 | "percent", 575 | "percent_rank", 576 | "percentile_cont", 577 | "percentile_disc", 578 | "placin", 579 | "plan", 580 | "pli", 581 | "position", 582 | "postfix", 583 | "power", 584 | "precedin", 585 | "precision", 586 | "prefix", 587 | "preorder", 588 | "prepare", 589 | "prepared", 590 | "preserve", 591 | "primary", 592 | "print", 593 | "prior", 594 | "privileges", 595 | "proc", 596 | "procedural", 597 | "procedure", 598 | "process", 599 | "processlist", 600 | "produce", 601 | "public", 602 | "purge", 603 | "quote", 604 | "raid0", 605 | "raiserror", 606 | "range", 607 | "rank", 608 | "ratchet", 609 | "raw", 610 | "read", 611 | "reads", 612 | "readtext", 613 | "real", 614 | "recheck", 615 | "reconfigure", 616 | "recursive", 617 | "ref", 618 | "references", 619 | "referencin", 620 | "regexp", 621 | "regr_avgx", 622 | "regr_avgy", 623 | "regr_count", 624 | "regr_intercept", 625 | "regr_r2", 626 | "regr_slope", 627 | "regr_sxx", 628 | "regr_sxy", 629 | "regr_syy", 630 | "reindex", 631 | "relative", 632 | "release", 633 | "reload", 634 | "rename", 635 | "repeat", 636 | "repeatable", 637 | "replace", 638 | "replication", 639 | "require", 640 | "reset", 641 | "resignal", 642 | "resource", 643 | "restart", 644 | "restore", 645 | "restrict", 646 | "result", 647 | "return", 648 | "returned_cardinality", 649 | "returned_length", 650 | "returned_octet_length", 651 | "returned_sqlstate", 652 | "returns", 653 | "revoke", 654 | "right", 655 | "rlike", 656 | "role", 657 | "rollback", 658 | "rollup", 659 | "routine", 660 | "routine_catalo", 661 | "routine_name", 662 | "routine_schema", 663 | "row", 664 | "row_count", 665 | "row_number", 666 | "rowcount", 667 | "rowguidcol", 668 | "rowid", 669 | "rownum", 670 | "rows", 671 | "rule", 672 | "save", 673 | "savepoint", 674 | "scale", 675 | "schema", 676 | "schema_name", 677 | "schemas", 678 | "scope", 679 | "scope_catalo", 680 | "scope_name", 681 | "scope_schema", 682 | "scroll", 683 | "search", 684 | "second", 685 | "second_microsecond", 686 | "section", 687 | "security", 688 | "select", 689 | "self", 690 | "sensitive", 691 | "separator", 692 | "sequence", 693 | "serializable", 694 | "server_name", 695 | "session", 696 | "session_user", 697 | "set", 698 | "setof", 699 | "sets", 700 | "setuser", 701 | "share", 702 | "show", 703 | "shutdown", 704 | "signal", 705 | "similar", 706 | "simple", 707 | "size", 708 | "smallint", 709 | "some", 710 | "soname", 711 | "source", 712 | "space", 713 | "spatial", 714 | "specific", 715 | "specific_name", 716 | "specifictype", 717 | "sql", 718 | "sql_big_result", 719 | "sql_big_selects", 720 | "sql_big_tables", 721 | "sql_calc_found_rows", 722 | "sql_log_off", 723 | "sql_log_update", 724 | "sql_low_priority_updates", 725 | "sql_select_limit", 726 | "sql_small_result", 727 | "sql_warnings", 728 | "sqlca", 729 | "sqlcode", 730 | "sqlerror", 731 | "sqlexception", 732 | "sqlstate", 733 | "sqlwarnin", 734 | "sqrt", 735 | "ssl", 736 | "stable", 737 | "start", 738 | "startin", 739 | "state", 740 | "statement", 741 | "static", 742 | "statistics", 743 | "status", 744 | "stddev_pop", 745 | "stddev_samp", 746 | "stdin", 747 | "stdout", 748 | "storage", 749 | "straight_join", 750 | "strict", 751 | "strin", 752 | "structure", 753 | "style", 754 | "subclass_origin", 755 | "sublist", 756 | "submultiset", 757 | "substrin", 758 | "successful", 759 | "sum", 760 | "superuser", 761 | "symmetric", 762 | "synonym", 763 | "sysdate", 764 | "sysid", 765 | "system", 766 | "system_user", 767 | "table", 768 | "table_name", 769 | "tables", 770 | "tablesample", 771 | "tablespace", 772 | "temp", 773 | "template", 774 | "temporary", 775 | "terminate", 776 | "terminated", 777 | "text", 778 | "textsize", 779 | "than", 780 | "then", 781 | "this", 782 | "ties", 783 | "time", 784 | "timestamp", 785 | "timezone_hour", 786 | "timezone_minute", 787 | "tinyblob", 788 | "tinyint", 789 | "tinytext", 790 | "to", 791 | "toast", 792 | "top", 793 | "top_level_count", 794 | "trailin", 795 | "tran", 796 | "transaction", 797 | "transaction_active", 798 | "transactions_committed", 799 | "transactions_rolled_back", 800 | "transform", 801 | "transforms", 802 | "translate", 803 | "translation", 804 | "treat", 805 | "trigger", 806 | "trigger_catalo", 807 | "trigger_name", 808 | "trigger_schema", 809 | "trim", 810 | "true", 811 | "truncate", 812 | "trusted", 813 | "tsequal", 814 | "type", 815 | "uescape", 816 | "uid", 817 | "unbounded", 818 | "uncommitted", 819 | "under", 820 | "undo", 821 | "unencrypted", 822 | "union", 823 | "unique", 824 | "unknown", 825 | "unlike", 826 | "unlisten", 827 | "unlock", 828 | "unnamed", 829 | "unnest", 830 | "unsigned", 831 | "until", 832 | "update", 833 | "updatetext", 834 | "upper", 835 | "usage", 836 | "use", 837 | "user", 838 | "user_defined_type_catalo", 839 | "user_defined_type_code", 840 | "user_defined_type_name", 841 | "user_defined_type_schema", 842 | "usin", 843 | "utc_date", 844 | "utc_time", 845 | "utc_timestamp", 846 | "vacuum", 847 | "valid", 848 | "validate", 849 | "validator", 850 | "value", 851 | "values", 852 | "var_pop", 853 | "var_samp", 854 | "varbinary", 855 | "varchar", 856 | "varchar2", 857 | "varcharacter", 858 | "variable", 859 | "variables", 860 | "varyin", 861 | "verbose", 862 | "view", 863 | "volatile", 864 | "waitfor", 865 | "when", 866 | "whenever", 867 | "where", 868 | "while", 869 | "width_bucket", 870 | "window", 871 | "with", 872 | "within", 873 | "without", 874 | "work", 875 | "write", 876 | "writetext", 877 | "x509", 878 | "xor", 879 | "year", 880 | "year_month", 881 | "zerofill", 882 | "zone" 883 | ]) 884 | 885 | if __name__ == "__main__": 886 | import doctest 887 | doctest.testmod() 888 | -------------------------------------------------------------------------------- /giles/pyre.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id: f27124002febf7a45c9611b27a258e913466cf4b $ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: Provide a simple tokenizer and operator precedence parser. 20 | # 21 | ###################################################################### 22 | 23 | """ 24 | pyre - Tokenize and Parse Expressions 25 | 26 | Pyre is a Python module that helps create tokenizers, parsers and 27 | interpreters for expressions - that is, collections of operators, operands, 28 | and function calls. 29 | 30 | Functionality is split into the Tokenizer and Parser classes. Either one can 31 | be used without the other, but they work quite well together. Tokenizer objects 32 | are iterators over strings that yield tokens, and Parser objects parse streams 33 | of tokens according to easily-defined precedence rules. 34 | 35 | Here's a simple example combining the Tokenizer class and the Parser class 36 | to produce a four-function calculator, plus the trigonometric function sin: 37 | 38 | Recall that the tokenizer just creates an iterator over the tokens: 39 | 40 | >>> class FourFunctionTokenizer(Tokenizer): 41 | ... # The self argument is an iterator, so not very useful - hence the 42 | ... # tokenizer argument. 43 | ... @syntax_rule(r"\\d+") 44 | ... def make_number(self, tokenizer, captures): 45 | ... return int(captures[0]) 46 | ... 47 | ... @syntax_rule(r"[-+*/]") 48 | ... def make_operator(self, tokenizer, captures): 49 | ... return OperatorToken(captures[0]) 50 | ... 51 | ... @syntax_rule(r"[a-z]+") 52 | ... def make_function(self, tokenizer, captures): 53 | ... return FunctionToken(captures[0]) 54 | ... 55 | >>> tokenizer = FourFunctionTokenizer() 56 | >>> for token in tokenizer.tokenize("1 + (2 * 3)"): 57 | ... print(token) 58 | 1 59 | + 60 | OPEN_PAREN 61 | 2 62 | * 63 | 3 64 | CLOSE_PAREN 65 | 66 | The tokenizer handles only a few things specially: parentheses, whitespace, 67 | and newlines. 68 | 69 | Whitespace and newlines can be replaced by overridding the newline and whitespace 70 | methods in the subclass. New forms of whitespace, such as comments, can be added 71 | easily by specifying the "skip" flag: 72 | 73 | >>> def make_comment(self, captures): 74 | ... pass 75 | ... 76 | >>> tokenizer.add_syntax(r"#[^\\n]*", make_comment, skip=True) 77 | >>> for token in tokenizer.tokenize('''# I'm a comment! 78 | ... 1 + (2 * 3)'''): 79 | ... print(token) 80 | NewlineToken() 81 | 1 82 | + 83 | OPEN_PAREN 84 | 2 85 | * 86 | 3 87 | CLOSE_PAREN 88 | 89 | Now, we create a simple parser that takes as its input the tokens produced by 90 | the tokenizer we defined: 91 | 92 | >>> import math 93 | >>> class FourFunctionParser(Parser): 94 | ... @binary_operator("+", precedence=10) 95 | ... def add(self, parser, arg1, arg2): 96 | ... return arg1 + arg2 97 | ... 98 | ... @binary_operator("-", precedence=10) 99 | ... def sub(self, parser, arg1, arg2): 100 | ... return arg1 - arg2 101 | ... 102 | ... @binary_operator("*", precedence=20) 103 | ... def mul(self, parser, arg1, arg2): 104 | ... return arg1 * arg2 105 | ... 106 | ... @binary_operator("/", precedence=20) 107 | ... def div(self, parser, arg1, arg2): 108 | ... return arg1 / arg2 109 | ... 110 | ... # Functions are just normal Python functions. They take two additional 111 | ... # arguments: self should be accepted but ignored (it is an iterator 112 | ... # object), and parser is the parser object. The remaining arguments, if 113 | ... # any, are the arguments passed to the function in the expression. 114 | ... @defined_function("sin") 115 | ... def sin(self, parser, arg1): 116 | ... return math.sin(arg1) 117 | ... 118 | >>> parser = FourFunctionParser() 119 | >>> tokenizer = FourFunctionTokenizer() 120 | >>> print(parser.parse(tokenizer.tokenize("1 + 2 * 3"))) 121 | 7 122 | 123 | The default implementation of the parser is stateless (though of course 124 | subclasses need not be): 125 | 126 | >>> print(parser.parse(tokenizer.tokenize("sin(1)"))) 127 | 0.8414709848078965 128 | 129 | Parentheses are handled appropriately: 130 | 131 | >>> print(parser.parse(tokenizer.tokenize("(1 + 2) * 3"))) 132 | 9 133 | """ 134 | 135 | __author__ = "Rob King" 136 | __copyright__ = "Copyright (C) 2011-2014 KoreLogic, Inc. All Rights Reserved." 137 | __credits__ = [] 138 | __license__ = "See README.LICENSE" 139 | __version__ = "$Id: f27124002febf7a45c9611b27a258e913466cf4b $" 140 | __maintainer__ = "Rob King" 141 | __email__ = "rking@korelogic.com" 142 | __status__ = "Alpha" 143 | 144 | import re 145 | 146 | 147 | def syntax_rule(regex, flags=0, skip=False): 148 | """ 149 | Decorate a function as a syntax rule for the Tokenizer class. 150 | 151 | This decorator can be used to decorate methods of subclasses of the 152 | Tokenizer class and declaratively add syntax. 153 | """ 154 | 155 | def wrap(func): 156 | def inner(self, tokenizer, groups): 157 | return func(self, tokenizer, groups) 158 | 159 | inner.syntax_rule = regex 160 | inner.syntax_flags = flags 161 | inner.syntax_skip = skip 162 | 163 | return inner 164 | return wrap 165 | 166 | 167 | class UnknownTokenException(Exception): 168 | 169 | """The UnknownTokenException indicates that untokenizable input was found 170 | and, if available, the input position of the untokenizable input.""" 171 | 172 | def __init__(self, token, near=None): 173 | super().__init__("Unknown input: '%s'%s" % (token, "" if near is None else " near '%s'" % near)) 174 | 175 | 176 | class Token: 177 | 178 | """ 179 | The Parser class expects its input in the form of a stream of Token objects 180 | and "other" objects. Token objects represent things that the parser cares. 181 | 182 | about - e.g. numbers, parentheses, function names. These are represented by 183 | different subclasses of the Token class. 184 | 185 | Everything else in the parser's input are "other" objects and can be 186 | anything. The parser just passes these objects to the user-defined operator 187 | functions, so it's up to the user to decide how to handle them. 188 | """ 189 | 190 | pass 191 | 192 | 193 | class OpenParenToken(Token): 194 | 195 | """ 196 | The OpenParenToken is one of the tokens the parser cares about. Parentheses 197 | are used to group expressions and alter default precedence, and surround 198 | arguments to functions. 199 | 200 | OpenParenToken objects represent, unsurprisingly, open (or left) 201 | parentheses. 202 | """ 203 | 204 | def __str__(self): 205 | return "OPEN_PAREN" 206 | 207 | 208 | class CloseParenToken(Token): 209 | 210 | """ 211 | The CloseParenToken is one of the tokens the parser cares about. 212 | Parentheses are used to group expressions and alter default precedence, and 213 | surround arguments to functions. 214 | 215 | CloseParenToken objects represent, unsurprisingly, close (or right) 216 | parentheses. 217 | """ 218 | 219 | def __str__(self): 220 | return "CLOSE_PAREN" 221 | 222 | 223 | class ArgumentSeparatorToken(Token): 224 | 225 | """ 226 | The ArgumentSeparatorToken class represents a separator between two 227 | arguments in a function invocation. 228 | 229 | By default, this is a comma. 230 | """ 231 | 232 | def __str__(self): 233 | return "ARGUMENT_SEPARATOR" 234 | 235 | 236 | class FunctionToken(Token): 237 | 238 | """ 239 | FunctionToken objects represent function names in input. 240 | 241 | The programmer is responsible for disambiguating these from "normal" 242 | variable names if such a concept is applicable in the program. 243 | """ 244 | 245 | def __init__(self, name): 246 | self.name = name 247 | 248 | 249 | class OperatorToken(Token): 250 | 251 | """ 252 | OperatorToken objects represents operators in input. 253 | 254 | They are handled by the parser according to the defined operator 255 | types. 256 | """ 257 | 258 | def __init__(self, name): 259 | self.name = name 260 | 261 | def __str__(self): 262 | return self.name 263 | 264 | 265 | class NewlineToken(Token): 266 | 267 | """NewlineToken objects are spat out by the tokenizer whenever a newline is 268 | encountered, allowing the parser to keep track of where it is in the input, 269 | if it is so inclined.""" 270 | 271 | def __str__(self): 272 | return "NewlineToken()" 273 | 274 | def __repr__(self): 275 | return str(self) 276 | 277 | 278 | class Tokenizer: 279 | 280 | """ 281 | A Tokenizer object implements the iterator protocol, and iterates over a 282 | string using a defined set of tokenization rules. 283 | 284 | Tokenizers are easily built using the syntax_rule decorator for methods. 285 | The Tokenizer class provides default rules for whitespace and newlines; these 286 | can be replaced by defining a whitespace or newline method in a subclass. 287 | There also exist default rules for parentheses and argument separators - this 288 | can be changed but probably shouldn't be. 289 | 290 | Rules can be added to the tokenizer after initialization using the add_syntax 291 | method. 292 | """ 293 | 294 | def __init__(self): 295 | self.rules = [] 296 | 297 | for attr in dir(self): 298 | obj = getattr(self, attr) 299 | if hasattr(obj, "syntax_rule"): 300 | self.add_syntax(obj.syntax_rule, obj, obj.syntax_flags, obj.syntax_skip) 301 | 302 | def tokenize(self, string): 303 | """Tokenize a string by returning an iterator over that string.""" 304 | 305 | class Iterator: 306 | 307 | def __init__(self, tokenizer, rules, string): 308 | self.rules = rules 309 | self.string = string 310 | self.tokenizer = tokenizer 311 | 312 | def __iter__(self): 313 | return self 314 | 315 | def __next__(self): 316 | longest_match = None 317 | longest_callback = None 318 | longest_length = 0 319 | longest_skip = False 320 | 321 | if len(self.string) == 0: 322 | raise StopIteration() 323 | 324 | for regex, callback, skip in self.rules: 325 | match = regex.match(self.string) 326 | if match: 327 | if len(match.group(0)) > longest_length: 328 | longest_match = match 329 | longest_callback = callback 330 | longest_length = len(match.group(0)) 331 | longest_skip = skip 332 | 333 | if longest_length == 0: 334 | raise UnknownTokenException(self.string) 335 | 336 | result = longest_callback(self.tokenizer, (longest_match.group(0),) + longest_match.groups()) 337 | self.string = self.string[longest_length:] 338 | 339 | if longest_skip: 340 | return next(self) 341 | 342 | else: 343 | return result 344 | 345 | return Iterator(self, self.rules, string) 346 | 347 | def add_syntax(self, regex, callback, flags=0, skip=False): 348 | """ 349 | Add a new tokenization rule. If the input matches the given regular 350 | expression (compiled with the optional flags), the captured groups are 351 | passed to the given callback, and the result of that callback is the 352 | token. 353 | 354 | If the skip flag is true, the tokenizer will execute the callback, but not 355 | return the token; it will instead iterate again. This is useful for things like 356 | comments and newlines. Note that, since the callback is still called, the 357 | rule can have side-effects such as updating the input position. 358 | """ 359 | 360 | self.rules.append((re.compile(regex, flags), callback, skip)) 361 | 362 | @syntax_rule(r"[ \t\r\v]+", skip=True) 363 | def whitespace(self, tokenizer, groups): 364 | pass 365 | 366 | @syntax_rule(r"\n") 367 | def newline(self, tokenizer, groups): 368 | return NewlineToken() 369 | 370 | @syntax_rule(r"[()]") 371 | def parentheses(self, tokenizer, groups): 372 | return OpenParenToken() if groups[0] == "(" else CloseParenToken() 373 | 374 | @syntax_rule(r"[,]") 375 | def separator(self, tokenizer, groups): 376 | return ArgumentSeparatorToken() 377 | 378 | 379 | class ExtraInputException(Exception): 380 | 381 | """The ExtraInputException indicates that there was extra input after the 382 | last element of the parsed expression.""" 383 | 384 | def __init__(self, extra): 385 | super().__init__("Extra input after end of expression: '%s'" % extra) 386 | self.extra = extra 387 | 388 | 389 | class MissingOperandException(Exception): 390 | 391 | """The MissingOperandException exception indicates that there were missing 392 | operands to some defined operator.""" 393 | 394 | def __init__(self, operator, line): 395 | super().__init__("Missing operand for '%s' operator on line %d" % (operator, line)) 396 | self.operator = operator 397 | 398 | 399 | class UnknownOperatorException(Exception): 400 | 401 | """The UnknownOperatorException indicates that the tokenizer passed an 402 | undefined operator to the parser.""" 403 | 404 | def __init__(self, operator, line): 405 | super().__init__("Unknown operator '%s' on %d" % (operator, line)) 406 | self.operator = operator 407 | 408 | 409 | class UnknownFunctionException(Exception): 410 | 411 | """The UnknownFunctionException indicates that the tokenizer passed an 412 | undefined function to the parser.""" 413 | 414 | def __init__(self, name, line): 415 | super().__init__("Unknown function '%s' on line %d" % (name, line)) 416 | self.function = name 417 | 418 | 419 | class MismatchedParenthesesException(Exception): 420 | 421 | """The MismatchedParenthesesException indicates a mismatched pair of 422 | parentheses were present in the expression.""" 423 | 424 | def __init__(self, line): 425 | super().__init__("Mismatched parentheses on line %d" % line) 426 | 427 | 428 | class Operator: 429 | 430 | """The Operator class represents an operator to be evaluated.""" 431 | 432 | def __init__(self, name, precedence, associativity): 433 | self.name = name 434 | self.precedence = precedence 435 | self.associativity = associativity 436 | 437 | def __str__(self): 438 | return "Operator(%s)" % (self.name,) 439 | 440 | def __repr__(self): 441 | return str(self) 442 | 443 | 444 | class BinaryOperator(Operator): 445 | 446 | """The BinaryOperator class represents a binary operator to be 447 | evaluated.""" 448 | 449 | def __init__(self, name, precedence, associativity, callback): 450 | super().__init__(name, precedence, associativity) 451 | self.callback = callback 452 | 453 | def __str__(self): 454 | return "BinaryOperator(%s)" % (self.name,) 455 | 456 | def __call__(self, parser, arg1, arg2): 457 | return self.callback(parser, arg1, arg2) 458 | 459 | 460 | class UnaryOperator(Operator): 461 | 462 | """The UnaryOperator class represents a unary operator to be evaluated.""" 463 | 464 | def __init__(self, name, precedence, associativity, callback): 465 | super().__init__(name, precedence, associativity) 466 | self.callback = callback 467 | 468 | def __str__(self): 469 | return "UnaryOperator(%s)" % (self.name,) 470 | 471 | def __call__(self, parser, arg1): 472 | return self.callback(parser, arg1) 473 | 474 | 475 | class Function: 476 | 477 | """The Function class represents a function known to the parser.""" 478 | 479 | def __init__(self, name, parser, callback): 480 | self.parser = parser 481 | self.callback = callback 482 | self.name = name 483 | 484 | def __str__(self): 485 | return "Function(%s)" % (self.name,) 486 | 487 | def __repr__(self): 488 | return str(self) 489 | 490 | 491 | class FunctionInvocation: 492 | 493 | def __init__(self, func, line=0): 494 | self.argcount = 0 495 | self.line = line 496 | self.name = func.name 497 | self.parser = func.parser 498 | self.callback = func.callback 499 | 500 | def __call__(self, stack): 501 | args = [self.parser.evaluate(stack, self.line) for i in range(0, self.argcount)] 502 | args.reverse() 503 | return self.callback(self.parser, *args) 504 | 505 | 506 | def defined_function(name=None): 507 | """ 508 | Decorate a function as a defined function known to the parser. 509 | 510 | This decorator can be used to decorate methods of subclasses of the 511 | Parser class and declaratively add functions. The name argument can 512 | be used to rename the function inside expressions. 513 | """ 514 | 515 | def wrap(func): 516 | def inner(self, parser, *args): 517 | return func(self, parser, *args) 518 | 519 | inner.defined_function_name = name if name is not None else func.__name__ 520 | 521 | return inner 522 | return wrap 523 | 524 | 525 | def binary_operator(name, precedence=10, associativity="left"): 526 | """ 527 | Decorate a function as a binary operator. 528 | 529 | This decorator can be used to decorate methods of subclasses of the 530 | Parser class and declaratively add new operators. 531 | """ 532 | 533 | def wrap(func): 534 | def inner(self, parser, arg1, arg2): 535 | return func(self, parser, arg1, arg2) 536 | 537 | inner.binary_operator_name = name 538 | inner.binary_operator_precedence = precedence 539 | inner.binary_operator_associativity = associativity 540 | 541 | return inner 542 | return wrap 543 | 544 | 545 | def unary_operator(name, precedence=10, associativity="right"): 546 | """ 547 | Decorate a function as a unary operator. 548 | 549 | This decorator can be used to decorate methods of subclasses of the 550 | Parser class and declaratively add new operators. 551 | """ 552 | 553 | def wrap(func): 554 | def inner(self, parser, arg1): 555 | return func(self, parser, arg1) 556 | 557 | inner.unary_operator_name = name 558 | inner.unary_operator_precedence = precedence 559 | inner.unary_operator_associativity = associativity 560 | 561 | return inner 562 | return wrap 563 | 564 | 565 | class Parser: 566 | 567 | """ 568 | A Parser object takes a stream of Token objects (retrieved by iterating 569 | over a Tokenizer object) and parses them according to the operator 570 | precedence rules defined for the parser. 571 | 572 | Rules are defined using the binary_operator, unary_operator, and 573 | defined_function decorators. Rules can be added after initialization 574 | time by using the add_binary_operator, add_unary_operator, and 575 | add_function methods. 576 | """ 577 | 578 | def __init__(self): 579 | self.binary_operators = {} 580 | self.unary_operators = {} 581 | self.functions = {} 582 | 583 | for attr in dir(self): 584 | obj = getattr(self, attr) 585 | if hasattr(obj, "defined_function_name"): 586 | self.add_function(obj.defined_function_name, obj) 587 | 588 | elif hasattr(obj, "binary_operator_name"): 589 | self.add_binary_operator(obj.binary_operator_name, obj.binary_operator_precedence, obj.binary_operator_associativity, obj) 590 | 591 | elif hasattr(obj, "unary_operator_name"): 592 | self.add_unary_operator(obj.unary_operator_name, obj.unary_operator_precedence, obj.unary_operator_associativity, obj) 593 | 594 | def add_binary_operator(self, name, precedence, associativity, callback): 595 | """ 596 | Add the named operator with the given precedence and associactivity. 597 | 598 | When the operator is evaluated, its arguments will be passed to 599 | the callback. 600 | """ 601 | 602 | assert precedence >= 0 603 | assert associativity in ("left", "right") 604 | 605 | self.binary_operators[name] = BinaryOperator(name, precedence, associativity, callback) 606 | 607 | def add_unary_operator(self, name, precedence, associativity, callback): 608 | """ 609 | Add the named operator with the given precedence and associactivity. 610 | 611 | When the operator is evaluated, its arguments will be passed to 612 | the callback. 613 | """ 614 | 615 | assert precedence >= 0 616 | assert associativity in ("left", "right") 617 | 618 | self.unary_operators[name] = UnaryOperator(name, precedence, associativity, callback) 619 | 620 | def add_function(self, name, callback): 621 | """Add the named function and associate it with the given callback.""" 622 | 623 | self.functions[name] = Function(name, self, callback) 624 | 625 | def parse(self, tokenizer): 626 | """ 627 | Parse the stream of tokens represented by the tokenizer and return the 628 | result of evaluating the stream. 629 | 630 | By defining operators that build nodes, "evaluating" can produce 631 | abstract syntax trees. 632 | """ 633 | 634 | line = 1 635 | last_token = None 636 | output = [] 637 | stack = [] 638 | 639 | for token in iter(tokenizer): 640 | if isinstance(token, NewlineToken): 641 | line = line + 1 642 | 643 | elif isinstance(token, FunctionToken): 644 | if token.name not in self.functions: 645 | raise UnknownFunctionException(token.name, line) 646 | 647 | stack.append(FunctionInvocation(self.functions[token.name], line)) 648 | 649 | elif isinstance(token, ArgumentSeparatorToken): 650 | if last_token is None or isinstance(last_token, Operator) or isinstance(last_token, OpenParenToken): 651 | raise MismatchedParenthesesException(line) 652 | 653 | while len(stack) > 0 and not isinstance(stack[-1], OpenParenToken): 654 | output.append(stack.pop()) 655 | 656 | if len(stack) < 2 or not isinstance(stack[-2], FunctionInvocation): 657 | raise MismatchedParenthesesException(line) 658 | 659 | if stack[-2].argcount == 0: 660 | stack[-2].argcount += 1 661 | 662 | stack[-2].argcount += 1 663 | 664 | elif isinstance(token, OperatorToken): 665 | operators = self.binary_operators 666 | if last_token is None or isinstance(last_token, OperatorToken) or isinstance(last_token, OpenParenToken): 667 | operators = self.unary_operators 668 | 669 | if token.name not in operators: 670 | raise UnknownOperatorException(token.name, line) 671 | 672 | o1 = operators[token.name] 673 | while len(stack) > 0 and isinstance(stack[-1], Operator) and \ 674 | ((o1.associativity == "left" and o1.precedence == stack[-1].precedence) or (o1.precedence < stack[-1].precedence)): 675 | output.append(stack.pop()) 676 | 677 | stack.append(o1) 678 | 679 | elif isinstance(token, OpenParenToken): 680 | stack.append(token) 681 | 682 | elif isinstance(token, CloseParenToken): 683 | if isinstance(last_token, ArgumentSeparatorToken): 684 | raise MismatchedParenthesesException(line) 685 | 686 | while len(stack) > 0 and not isinstance(stack[-1], OpenParenToken): 687 | output.append(stack.pop()) 688 | 689 | if len(stack) == 0: 690 | raise MismatchedParenthesesException(line) 691 | 692 | stack.pop() 693 | 694 | if len(stack) > 0 and isinstance(stack[-1], FunctionInvocation): 695 | func = stack.pop() 696 | if (not isinstance(last_token, OpenParenToken)) and func.argcount == 0: 697 | func.argcount = 1 698 | output.append(func) 699 | 700 | else: 701 | if last_token is not None and not (isinstance(last_token, Operator) or isinstance(last_token, Token)): 702 | raise ExtraInputException(str(token)) 703 | output.append(token) 704 | 705 | last_token = token 706 | 707 | while len(stack) > 0: 708 | if isinstance(stack[-1], OpenParenToken) or isinstance(stack[-1], CloseParenToken): 709 | raise MismatchedParenthesesException(line) 710 | 711 | output.append(stack.pop()) 712 | 713 | result = self.evaluate(output, line) 714 | if len(output) > 0: 715 | raise ExtraInputException(output) 716 | 717 | return result 718 | 719 | def evaluate(self, stack, line=0): 720 | """Evaluate the operator or function on the top of the stack.""" 721 | 722 | operator = stack.pop() 723 | 724 | if isinstance(operator, UnaryOperator): 725 | if len(stack) < 1: 726 | raise MissingOperandException(operator.name, line) 727 | 728 | arg1 = self.evaluate(stack, line) 729 | return operator(self, arg1) 730 | 731 | elif isinstance(operator, BinaryOperator): 732 | if len(stack) < 2: 733 | raise MissingOperandException(operator.name, line) 734 | 735 | arg2 = self.evaluate(stack, line) 736 | arg1 = self.evaluate(stack, line) 737 | return operator(self, arg1, arg2) 738 | 739 | elif isinstance(operator, FunctionInvocation): 740 | return operator(stack) 741 | 742 | else: 743 | return operator 744 | 745 | if __name__ == "__main__": 746 | import doctest 747 | doctest.testmod() 748 | -------------------------------------------------------------------------------- /giles/sqlite_backend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id: 86ab295a13893bedc328583380980f633ff0ef52 $ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: Generate a SQLite database schema from a Giles engine 20 | # description. 21 | # 22 | ###################################################################### 23 | 24 | """Giles backend for SQLite.""" 25 | 26 | __author__ = "Rob King" 27 | __copyright__ = "Copyright (C) 2011-2014 KoreLogic, Inc. All Rights Reserved." 28 | __credits__ = [] 29 | __license__ = "See README.LICENSE." 30 | __version__ = "$Id: 86ab295a13893bedc328583380980f633ff0ef52 $" 31 | __maintainer__ = "Rob King" 32 | __email__ = "rking@korelogic.com" 33 | __status__ = "Alpha" 34 | 35 | import datetime 36 | import jinja2 37 | 38 | from pkg_resources import resource_string 39 | from giles import expression 40 | from giles.caseless_string import CaselessString as CS 41 | from itertools import chain 42 | 43 | ###################################################################### 44 | # 45 | # Globals used in this module. 46 | # 47 | ###################################################################### 48 | 49 | domains = {} # Domains for the only_once routines 50 | prefix = "giles" # Default prefix for object names 51 | indexes = {} # The indexes on each fact table 52 | 53 | ###################################################################### 54 | # 55 | # Transform a Giles expression into a SQL expression. 56 | # 57 | ###################################################################### 58 | 59 | 60 | def generate_expression(value, fact, frame_prefix=None, fact_prefix=None): 61 | """ 62 | value - the expression or value to SQLize 63 | fact - the fact being matched 64 | frame_prefix - frame prefix 65 | fact_prefix - fact prefix 66 | """ 67 | 68 | if type(value) in (float, int): 69 | return str(value) 70 | 71 | elif type(value) == bool: 72 | return "1" if value else "0" 73 | 74 | elif type(value) == str: 75 | return "'%s'" % value.replace("'", "''") 76 | 77 | elif isinstance(value, expression.ThisReferenceNode): 78 | return '%s%s' % ((fact_prefix + '.') if fact_prefix else '', value.variable) 79 | 80 | elif isinstance(value, expression.LocalReferenceNode): 81 | return '%s%s' % ((frame_prefix + '.') if frame_prefix else '', value.variable) 82 | 83 | elif isinstance(value, expression.BinaryOpNode): 84 | return "(%s) %s (%s)" % (generate_expression(value.arg1, fact, frame_prefix, fact_prefix), 85 | value.operation, 86 | generate_expression(value.arg2, fact, frame_prefix, fact_prefix)) 87 | 88 | elif isinstance(value, expression.UnaryOpNode): 89 | return "(%s(%s))" % (value.operation, generate_expression(value.arg1, fact, frame_prefix, fact_prefix)) 90 | 91 | elif isinstance(value, expression.IfNode): 92 | return "(CASE WHEN (%s) THEN (%s) ELSE (%s) END)" % (generate_expression(value.predicate, fact, frame_prefix, fact_prefix), 93 | generate_expression(value.if_true, fact, frame_prefix, fact_prefix), 94 | generate_expression(value.if_false, fact, frame_prefix, fact_prefix)) 95 | 96 | elif isinstance(value, expression.FunctionNode): 97 | return "%s(%s)" % (value.external, ",".join(generate_expression(x, fact, frame_prefix, fact_prefix) for x in value.args)) 98 | 99 | elif isinstance(value, expression.CastNode): 100 | kind = {bool: "integer", int: "integer", float: "real", str: "text"}[value.type] 101 | return "CAST((%s) AS %s)" % (generate_expression(value.expression, fact, frame_prefix, fact_prefix), kind) 102 | 103 | elif isinstance(value, expression.JoinNode): 104 | tests = [] 105 | equality_tests = [] 106 | inequality_tests = [] 107 | 108 | def flatten(node): 109 | if not isinstance(node, expression.JoinNode): 110 | tests.append(node) 111 | 112 | else: 113 | flatten(node.left) 114 | flatten(node.right) 115 | 116 | flatten(value) 117 | 118 | for test in tests: 119 | if isinstance(test, expression.BinaryOpNode) and test.operation == "=": 120 | equality_tests.append(generate_expression(test, fact, frame_prefix, fact_prefix)) 121 | 122 | else: 123 | inequality_tests.append(generate_expression(test, fact, frame_prefix, fact_prefix)) 124 | 125 | return " AND ".join(equality_tests + inequality_tests) 126 | 127 | assert False 128 | 129 | ###################################################################### 130 | # 131 | # Generate a predicate for a match. This takes all the constant 132 | # tests against a given match and applies them. 133 | # 134 | ###################################################################### 135 | 136 | 137 | def flatten_predicate(when): 138 | predicates = [] 139 | 140 | if isinstance(when, expression.JoinNode): 141 | predicates += flatten_predicate(when.left) 142 | predicates += flatten_predicate(when.right) 143 | 144 | elif isinstance(when, expression.BinaryOpNode) and when.type == bool and \ 145 | isinstance(when.arg1, expression.ThisReferenceNode) and len(find_locals(when.arg2)) == 0: 146 | predicates.append(when) 147 | 148 | return predicates 149 | 150 | 151 | def generate_predicate_wrapper(fact, when): 152 | results = generate_predicate(fact, when) 153 | if len(results) == 0: 154 | return "1" 155 | else: 156 | return " AND ".join(results) 157 | 158 | 159 | def generate_predicate(fact, when): 160 | """ 161 | fact - the fact being matched 162 | when - the predicate 163 | """ 164 | 165 | tests = flatten_predicate(when) 166 | tests.sort(key=lambda x: str(x.arg1.variable).lower()) 167 | 168 | return [generate_expression(x, fact, None, 'new') for x in tests] 169 | 170 | ###################################################################### 171 | # 172 | # Generate a list of locals used in an expression. 173 | # 174 | ###################################################################### 175 | 176 | 177 | def find_locals(value): 178 | if isinstance(value, expression.LocalReferenceNode): 179 | return [value.variable] 180 | 181 | elif isinstance(value, expression.BinaryOpNode): 182 | return find_locals(value.arg1) + find_locals(value.arg2) 183 | 184 | elif isinstance(value, expression.UnaryOpNode): 185 | return find_locals(value.arg1) 186 | 187 | elif isinstance(value, expression.IfNode): 188 | return find_locals(value.predicate) + find_locals(value.if_true) + find_locals(value.if_false) 189 | 190 | elif isinstance(value, expression.FunctionNode): 191 | local_vars = [] 192 | for arg in value.args: 193 | local_vars += find_locals(arg) 194 | return local_vars 195 | 196 | elif isinstance(value, expression.CastNode): 197 | return find_locals(value.expression) 198 | 199 | elif isinstance(value, expression.JoinNode): 200 | return find_locals(value.left) + find_locals(value.right) 201 | 202 | else: 203 | return [] 204 | 205 | ###################################################################### 206 | # 207 | # Add an index for a table. If an index already exists for that table 208 | # that is a superset of this index, there is no need to generate a new 209 | # index; the longer one will be used. Note that for the purposes of 210 | # determining the superset relation, field order is significant 211 | # (because SQLite's query optimizer will only apply an index if the 212 | # list of indexed fields is in the same order as the fields were 213 | # specified in the query). 214 | # 215 | ###################################################################### 216 | 217 | 218 | def add_index(table, fields): 219 | table = table.lower() 220 | 221 | if len(fields) == 0: 222 | return 223 | 224 | if table not in indexes: 225 | indexes[table] = {} 226 | 227 | current = indexes[table] 228 | for field in fields: 229 | if field not in current: 230 | current[field] = {} 231 | current = current[field] 232 | 233 | ###################################################################### 234 | # 235 | # Generate a join expression. This takes all of the tests against 236 | # a given fact and applies them (optionally excluding the constant 237 | # tests). Equalities are sorted and placed in front of any 238 | # inequalities, which allows for better index usage and query 239 | # optimization. 240 | # 241 | ###################################################################### 242 | 243 | 244 | def generate_join(when, fact, frame_prefix=None, fact_prefix=None, include_constants=True): 245 | result = "" 246 | equalities = [] 247 | inequalities = [] 248 | 249 | if fact_prefix is not None: 250 | fact_prefix = fact_prefix.lower() 251 | 252 | if frame_prefix is not None: 253 | frame_prefix = frame_prefix.lower() 254 | 255 | def flatten(n): 256 | if isinstance(n, expression.JoinNode): 257 | flatten(n.left) 258 | flatten(n.right) 259 | 260 | elif isinstance(n, expression.BinaryOpNode) and n.type == bool and isinstance(n.arg1, expression.ThisReferenceNode): 261 | if type(n.arg2) not in (bool, float, int, str) or include_constants: 262 | if n.operation == "=": 263 | equalities.append(n) 264 | 265 | else: 266 | inequalities.append(n) 267 | 268 | flatten(when) 269 | 270 | equalities.sort(key=lambda x: str(x.arg1.variable).lower()) 271 | inequalities.sort(key=lambda x: str(x.arg1.variable).lower()) 272 | 273 | equality_predicate = " AND ".join([generate_expression(predicate, fact, frame_prefix, fact_prefix).strip() for predicate in equalities]) 274 | inequality_predicate = " AND ".join( 275 | [generate_expression(predicate, fact, frame_prefix, fact_prefix).strip() for predicate in inequalities]) 276 | 277 | result = equality_predicate + \ 278 | (" AND " if len(equality_predicate.strip()) and len(inequality_predicate.strip()) else "") + inequality_predicate 279 | if len(result.strip()) == 0: 280 | return None 281 | 282 | if frame_prefix not in ('new', 'old'): # Don't add indexes for expressions over immediately-available data (i.e. new/old frames/facts) 283 | equality_variables = [] 284 | for predicate in equalities: 285 | equality_variables += find_locals(predicate) 286 | 287 | if len(inequalities) > 0: 288 | add_index(frame_prefix, equality_variables + find_locals(inequalities[0])) 289 | 290 | elif len(equality_variables) > 0: 291 | add_index(frame_prefix, equality_variables) 292 | 293 | if fact_prefix not in ('new', 'old'): # Don't add indexes for expressions over immediately-available data (i.e. new/old frames/facts) 294 | equality_variables = [] 295 | for predicate in equalities: 296 | equality_variables.append(predicate.arg1.variable) 297 | 298 | if len(inequalities) > 0: 299 | add_index(fact_prefix, equality_variables + [inequalities[0].arg1.variable]) 300 | 301 | elif len(equality_variables) > 0: 302 | add_index(fact_prefix, equality_variables) 303 | 304 | return result 305 | 306 | ###################################################################### 307 | # 308 | # Print out a value only when called with it the first time. 309 | # Subsequent times, print out nothing. "Once" is determined by a 310 | # combination of the value to print and the "domain" in which it is 311 | # printing. 312 | # 313 | ###################################################################### 314 | 315 | 316 | def only_once(domain, value): 317 | domain = domain.lower() 318 | 319 | if domain not in domains: 320 | domains[domain] = {} 321 | 322 | if value not in domains[domain]: 323 | domains[domain][value] = True 324 | return value 325 | 326 | return '' 327 | 328 | ###################################################################### 329 | # 330 | # Generate a synthetic assignment for predicates specified over locals 331 | # in later match clauses. 332 | # 333 | ###################################################################### 334 | 335 | 336 | def flatten_local_predicates(when): 337 | predicates = [] 338 | 339 | if isinstance(when, expression.JoinNode): 340 | predicates += flatten_local_predicates(when.left) 341 | predicates += flatten_local_predicates(when.right) 342 | 343 | elif isinstance(when, expression.BinaryOpNode) and when.type == bool and \ 344 | isinstance(when.arg1, expression.ThisReferenceNode) and len(find_locals(when.arg2)) > 0: 345 | predicates.append(when) 346 | 347 | return predicates 348 | 349 | 350 | def immediate_substitute(predicate, used_var, actual): 351 | if isinstance(predicate, expression.BinaryOpNode): 352 | if isinstance(predicate.arg1, expression.LocalReferenceNode) and predicate.arg1.variable == used_var: 353 | predicate.arg1 = actual 354 | 355 | else: 356 | immediate_substitute(predicate.arg1, used_var, actual) 357 | 358 | if isinstance(predicate.arg2, expression.LocalReferenceNode) and predicate.arg2.variable == used_var: 359 | predicate.arg2 = actual 360 | 361 | else: 362 | immediate_substitute(predicate.arg2, used_var, actual) 363 | 364 | elif isinstance(predicate, expression.UnaryOpNode): 365 | if isinstance(predicate.arg1, expression.LocalReferenceNode) and predicate.arg1.variable == used_var: 366 | predicate.arg1 = actual 367 | 368 | else: 369 | immediate_substitute(predicate.arg1, used_var, actual) 370 | 371 | elif isinstance(predicate, expression.IfNode): 372 | if isinstance(predicate.predicate, expression.LocalReferenceNode) and predicate.predicate.variable == used_var: 373 | predicate.predicate = actual 374 | 375 | else: 376 | immediate_substitute(predicate.predicate, used_var, actual) 377 | 378 | if isinstance(predicate.if_true, expression.LocalReferenceNode) and predicate.if_true.variable == used_var: 379 | predicate.if_true = actual 380 | 381 | else: 382 | immediate_substitute(predicate.if_true, used_var, actual) 383 | 384 | if isinstance(predicate.if_false, expression.LocalReferenceNode) and predicate.if_false.variable == used_var: 385 | predicate.if_false = actual 386 | 387 | else: 388 | immediate_substitute(predicate.if_false, used_var, actual) 389 | 390 | elif isinstance(predicate, expression.FunctionNode): 391 | for i, arg in enumerate(predicate.args): 392 | if isinstance(arg, expression.LocalReferenceNode) and arg.variable == used_var: 393 | predicate.args[i] = actual 394 | 395 | else: 396 | immediate_substitute(predicate.args[i], used_var, actual) 397 | 398 | elif isinstance(predicate, expression.CastNode): 399 | if isinstance(predicate.expression, expression.LocalReferenceNode) and predicate.expression.variable == used_var: 400 | predicate.expression = actual 401 | 402 | else: 403 | immediate_substitute(predicate.expression, used_var, actual) 404 | 405 | 406 | def generate_synthetic_assignment(predicate, rule, prev_clause, synthetic_name): 407 | synthetic_name = CS(synthetic_name) 408 | if "locals" not in rule: 409 | rule["locals"] = {} 410 | 411 | if "assignments" not in prev_clause: 412 | prev_clause["assignments"] = {} 413 | 414 | rule["locals"][synthetic_name] = predicate.type 415 | 416 | used_vars = set(find_locals(predicate)) 417 | for used_var in used_vars.intersection(prev_clause["assignments"]): 418 | immediate_substitute(predicate, used_var, prev_clause["assignments"][used_var]) 419 | 420 | prev_clause["assignments"][synthetic_name] = predicate 421 | 422 | ###################################################################### 423 | # 424 | # Generate the SQL. 425 | # 426 | ###################################################################### 427 | 428 | 429 | def generate(new_prefix, filename, description, facts, parameters, rules): 430 | #################################################################### 431 | # 432 | # Reset global state. 433 | # 434 | #################################################################### 435 | 436 | global domains 437 | global prefix 438 | global indexes 439 | 440 | domains = {} 441 | prefix = "giles" 442 | indexes = {} 443 | 444 | #################################################################### 445 | # 446 | # Set global options. 447 | # 448 | #################################################################### 449 | 450 | prefix = "_" + new_prefix 451 | 452 | #################################################################### 453 | # 454 | # Mark as output any fact with an always-true predicate. 455 | # This saves a lot of time on the alpha pruning phase. 456 | # 457 | #################################################################### 458 | 459 | class OutputFact(dict): 460 | is_output = True 461 | 462 | for rule_clause in rules.values(): 463 | for match in rule_clause["matches"] + rule_clause["inverted_matches"]: 464 | predicate = generate_predicate(match["fact"], match["when"]) 465 | if len(predicate) == 0: 466 | facts[CS(match["fact"])] = OutputFact(facts[CS(match["fact"])]) 467 | 468 | #################################################################### 469 | # 470 | # Optimize any match clauses that have expressions using assignments 471 | # from earlier match clauses. 472 | # 473 | #################################################################### 474 | 475 | synthetic_count = 0 476 | for rule_name, rule in rules.items(): # Walk through each rule. 477 | for i, match_clause in enumerate(chain(rule["matches"], rule["inverted_matches"])): # Walk through each match subclause. 478 | if match_clause["when"] is not None: # If that subclause has a complex predicate. 479 | clauses = flatten_local_predicates(match_clause["when"]) # Grab the parts of that predicate. 480 | for clause in clauses: # Walk through those parts. 481 | if len(set(find_locals(clause))): # If that part uses local variables. 482 | if not isinstance(clause.arg2, expression.LocalReferenceNode): # And it's part of a complex expression. 483 | synthetic_count += 1 # We know we'll need a synthetic assign. 484 | synthetic_name = "synthetic_assignment_%d" % synthetic_count # Create a name for it. 485 | prev_i = min(len(rule["matches"]) - 1, max(i - 1, 0)) # Grab the previous assignable clause's index. 486 | prev_clause = rule["matches"][prev_i] # And grab the clause itself. 487 | generate_synthetic_assignment(clause.arg2, rule, prev_clause, synthetic_name) 488 | clause.arg2 = expression.LocalReferenceNode(synthetic_name, clause.arg2.type) 489 | 490 | #################################################################### 491 | # 492 | # Open the template and run it. 493 | # 494 | #################################################################### 495 | 496 | template_file = resource_string(__name__, 'sqlite.jinja').decode('utf-8') 497 | env = jinja2.Environment(loader=jinja2.FunctionLoader(lambda x: (template_file, 'sqlite.jinja', lambda: template_file))) 498 | template = env.get_template('sqlite.jinja') 499 | 500 | names = { 501 | "generate_join": generate_join, 502 | "generate_expression": generate_expression, 503 | "generate_predicate": generate_predicate_wrapper, 504 | "description": description, 505 | "facts": facts, 506 | "file": filename, 507 | "parameters": parameters, 508 | "only_once": only_once, 509 | "prefix": prefix, 510 | "public_prefix": new_prefix, 511 | "rules": rules, 512 | "bool": bool, 513 | "int": int, 514 | "float": float, 515 | "str": str, 516 | "time": str(datetime.datetime.now()) 517 | } 518 | 519 | result = template.render(**names) 520 | 521 | #################################################################### 522 | # 523 | # Spit out all the automatically-created indexes. 524 | # 525 | #################################################################### 526 | 527 | index_number = 0 528 | 529 | def dft(tree, leaf_callback, path=None): 530 | path = [] if path is None else path 531 | if len(tree) == 0: 532 | leaf_callback(path) 533 | 534 | else: 535 | for k, v in tree.items(): 536 | dft(v, leaf_callback, path + [k]) 537 | 538 | def callback(table, path): 539 | nonlocal index_number 540 | nonlocal result 541 | 542 | index_number += 1 543 | result += "\nCREATE INDEX %s_auto_index_%d ON %s(%s);" % (prefix, index_number, table, ",".join(path)) 544 | 545 | for table, tree in indexes.items(): 546 | dft(tree, lambda path: callback(table, path)) 547 | 548 | return result 549 | -------------------------------------------------------------------------------- /giles/validate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id: 12aa7938934929a1a4ba7bee92bcdceb8c9e391f $ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: Validate data structures against a schema. 20 | # 21 | ###################################################################### 22 | 23 | """ 24 | validate - Validate Arbitrary Datastructures 25 | 26 | This module provides a set of combinators that can be used to validate data 27 | structures of arbitrary shape and size. For example: 28 | 29 | >>> validator = Dictionary( 30 | ... required = { 31 | ... "First" : String(min_length=1, max_length=22), 32 | ... "Last" : String(min_length=1, max_length=22), 33 | ... }, 34 | ... optional = { 35 | ... "Middle" : String(min_length=1, max_length=22), 36 | ... }, 37 | ... case_sensitive=False 38 | ... ) 39 | >>> validator({"first" : "Rob", "LAST" : "King"}) is not None 40 | True 41 | """ 42 | 43 | __author__ = "Rob King" 44 | __copyright__ = "Copyright (C) 2011-2014 KoreLogic, Inc. All Rights Reserved." 45 | __credits__ = [] 46 | __license__ = "See README.LICENSE." 47 | __version__ = "$Id: 12aa7938934929a1a4ba7bee92bcdceb8c9e391f $" 48 | __maintainer__ = "Rob King" 49 | __email__ = "rking@korelogic.com" 50 | __status__ = "Alpha" 51 | 52 | import collections 53 | from giles.caseless_string import CaselessString as CS 54 | 55 | 56 | class ValidationException(Exception): 57 | 58 | def __init__(self, message="Invalid data", expected=None, actual=None, location="???"): 59 | super().__init__(message) 60 | 61 | self.message = message 62 | self.expected = expected 63 | self.actual = actual 64 | self.location = location 65 | 66 | def __str__(self): 67 | return "%s: %s\nExpected: %s\nActual: %s" % (self.location, self.message, self.expected, self.actual) 68 | 69 | 70 | class Validator: 71 | 72 | """The Validator class is the superclass of all validators.""" 73 | 74 | def __call__(self, obj): 75 | return obj 76 | 77 | def __str__(self): 78 | return repr(self) 79 | 80 | 81 | class All(Validator): 82 | 83 | """Validate a structure against multiple predicates, matching all of 84 | them.""" 85 | 86 | def __init__(self, *validators): 87 | self.validators = validators 88 | 89 | def __call__(self, obj, location="/"): 90 | for validator in self.validators: 91 | validator(obj, location) 92 | 93 | return obj 94 | 95 | def __repr__(self): 96 | return "All(%s)" % ", ".join(str(x) for x in self.validators) 97 | 98 | 99 | class Any(Validator): 100 | 101 | """Validate a structure against multiple predicates, matching at least one 102 | of them.""" 103 | 104 | def __init__(self, *validators): 105 | self.validators = validators 106 | 107 | def __call__(self, obj, location="/"): 108 | exceptions = [] 109 | for validator in self.validators: 110 | try: 111 | return validator(obj, location) 112 | 113 | except Exception as e: 114 | exceptions.append(e) 115 | 116 | raise ValidationException(" and ".join([x.message for x in exceptions]), str(self), obj, location) 117 | 118 | def __repr__(self): 119 | return "Any(%s)" % ", ".join(str(x) for x in self.validators) 120 | 121 | 122 | class Boolean(Validator): 123 | 124 | """Validate a structure as a boolean value.""" 125 | 126 | def __init__(self, value=None): 127 | self.value = value 128 | 129 | def __call__(self, obj, location="/"): 130 | if not isinstance(obj, bool): 131 | raise ValidationException("Expected a boolean value", bool, type(obj), location) 132 | 133 | if self.value is not None: 134 | if self.value != obj: 135 | raise ValidationException("Expected a specific boolean value", self.value, obj, location) 136 | 137 | return obj 138 | 139 | 140 | class Not(Validator): 141 | 142 | """Validate a structure by specifying what it shouldn't be.""" 143 | 144 | def __init__(self, other, message): 145 | self.other = other 146 | self.message = message 147 | 148 | def __call__(self, obj, location="/"): 149 | try: 150 | self.other(obj, location) 151 | 152 | except: 153 | return obj 154 | 155 | raise ValidationException(self.message, "Not %s" % self.other, str(obj), location) 156 | 157 | def __repr__(self): 158 | return "Not(%s)" % self.other 159 | 160 | 161 | class Dictionary(Validator): 162 | 163 | """Validate a dictionary.""" 164 | 165 | def casefold(self, s): 166 | if isinstance(s, str) and not self.case_sensitive: 167 | return CS(s) 168 | return s 169 | 170 | def __init__(self, required=None, optional=None, extra=None, extra_keys=None, min_extra=None, max_extra=None, case_sensitive=True, 171 | allow_dups=False, key_type=str, ordered=False): 172 | self.case_sensitive = case_sensitive 173 | self.required = {self.casefold(k): v for k, v in required.items()} if required is not None else {} 174 | self.optional = {self.casefold(k): v for k, v in optional.items()} if optional is not None else {} 175 | self.extra = extra 176 | self.min_extra = min_extra 177 | self.max_extra = max_extra 178 | self.extra_keys = extra_keys 179 | self.allow_duplicates = allow_dups 180 | self.key_type = key_type 181 | self.ordered = ordered 182 | 183 | def __call__(self, obj, location="/"): 184 | if not isinstance(obj, dict): 185 | raise ValidationException("Expected a dictionary", self, obj, location) 186 | 187 | extra_count = 0 188 | keys = [self.casefold(x) for x in obj.keys()] 189 | dummy = {self.casefold(k): v for k, v in obj.items()} 190 | 191 | result = {} if not self.ordered else collections.OrderedDict() 192 | stringify = CS if not self.case_sensitive else str 193 | 194 | for key in self.required.keys(): 195 | if key not in keys: 196 | raise ValidationException("Missing required key '%s'" % key, key, None, location) 197 | 198 | for key in keys: 199 | if not isinstance(key, self.key_type): 200 | raise ValidationException("Invalid key type", self.key_type, type(key), location) 201 | 202 | if not self.allow_duplicates and keys.count(key) > 1: 203 | raise ValidationException("Duplicate keys '%s'" % key, None, None, location) 204 | 205 | if not self.extra and key not in self.required.keys() and key not in self.optional.keys(): 206 | raise ValidationException("Disallowed key '%s'" % key, None, key, location) 207 | 208 | if key in self.required.keys(): 209 | result[stringify(key)] = self.required[key](dummy[key], "%s[%s]" % (location, key)) 210 | 211 | elif key in self.optional.keys(): 212 | result[stringify(key)] = self.optional[key](dummy[key], "%s[%s]" % (location, key)) 213 | 214 | else: 215 | extra_count += 1 216 | 217 | if self.extra_keys is not None: 218 | self.extra_keys(key, "%s[%s]" % (location, key)) 219 | 220 | if self.extra is not None: 221 | result[stringify(key)] = self.extra(dummy[key], "%s[%s]" % (location, key)) 222 | 223 | if self.min_extra is not None and extra_count < self.min_extra: 224 | raise ValidationException("Expected at least %d extra keys" % self.min_extra, self.min_extra, extra_count, location) 225 | 226 | if self.max_extra is not None and extra_count > self.max_extra: 227 | raise ValidationException("Expected at most %d extra keys" % self.max_extra, self.max_extra, extra_count, location) 228 | 229 | return result 230 | 231 | def __repr__(self): 232 | return "Dictionary(required=%s, optional=%s, extra=%s, extra_keys=%s, case_sensitive=%s, allow_duplicates=%s, key_type=%s)" % ( 233 | self.required, 234 | self.optional, 235 | self.extra, 236 | self.extra_keys, 237 | self.case_sensitive, 238 | self.allow_duplicates, 239 | self.key_type) 240 | 241 | 242 | class Float(Validator): 243 | 244 | """Validate an integer.""" 245 | 246 | def __init__(self, value=None, minimum=None, maximum=None, allow_integer=True): 247 | self.minimum = minimum 248 | self.maximum = maximum 249 | self.allow_integer = allow_integer 250 | self.value = value 251 | 252 | def __call__(self, obj, location="/"): 253 | if not isinstance(obj, float): 254 | raise ValidationException("Expected a float", float, type(obj), location) 255 | 256 | if isinstance(obj, int) and not self.allow_integer: 257 | raise ValidationException("Expceted a float", float, type(obj), location) 258 | 259 | if self.value is not None and obj != self.value: 260 | raise ValidationException("Expected %g" % self.value, self.value, obj, location) 261 | 262 | if self.minimum is not None and obj < self.minimum: 263 | raise ValidationException("Expected a float greater than %d" % self.minimum, self.minimum, obj, location) 264 | 265 | if self.maximum is not None and obj > self.maximum: 266 | raise ValidationException("Expected a float less than %d" % self.maximum, self.maximum, obj, location) 267 | 268 | return obj 269 | 270 | def __repr__(self): 271 | return "Float(%s, minimum=%s, maximum=%s, allow_integer=%s" % (self.value, self.minimum, self.maximum, self.allow_integer) 272 | 273 | 274 | class InstanceOf(Validator): 275 | 276 | """Ensure that some object is an instance of a class.""" 277 | 278 | def __init__(self, kind): 279 | self.kind = kind 280 | 281 | def __call__(self, obj, location="/"): 282 | if not isinstance(obj, self.kind): 283 | raise ValidationException("Expected an object of type '%s'" % self.kind, self.kind, type(obj), location) 284 | 285 | return obj 286 | 287 | def __repr__(self): 288 | return "InstanceOf(%s)" % self.kind 289 | 290 | 291 | class Integer(Validator): 292 | 293 | """Validate an integer.""" 294 | 295 | def __init__(self, value=None, minimum=None, maximum=None, allow_bool=False): 296 | self.value = value 297 | self.minimum = minimum 298 | self.maximum = maximum 299 | self.allow_bool = allow_bool 300 | 301 | def __call__(self, obj, location="/"): 302 | if not isinstance(obj, int): 303 | raise ValidationException("Expected an integer", int, type(obj), location) 304 | 305 | if isinstance(obj, bool) and not self.allow_bool: 306 | raise ValidationException("Expected an integer", int, type(obj), location) 307 | 308 | if self.value is not None and obj != self.value: 309 | raise ValidationException("Expected %d" % self.value, self.value, obj, location) 310 | 311 | if self.minimum is not None and obj < self.minimum: 312 | raise ValidationException("Expected an integer greater than %d" % self.minimum, self.minimum, obj, location) 313 | 314 | if self.maximum is not None and obj > self.maximum: 315 | raise ValidationException("Expected an integer less than %d" % self.maximum, self.maximum, obj, location) 316 | 317 | return obj 318 | 319 | def __repr__(self): 320 | return "Integer(%s, minimum=%s, maximum=%s, allow_bool=%s" % (self.value, self.minimum, self.maximum, self.allow_bool) 321 | 322 | 323 | class List(Validator): 324 | 325 | """Validate a list.""" 326 | 327 | def __init__(self, members, min_length=None, max_length=None): 328 | self.min_length = min_length 329 | self.max_length = max_length 330 | self.members = members 331 | 332 | def __call__(self, obj, location="/"): 333 | if not isinstance(obj, list): 334 | raise ValidationException("Expected a list", self, type(obj), location) 335 | 336 | if self.min_length is not None and len(obj) < self.min_length: 337 | raise ValidationException("Expected a list of at least length %d" % self.min_length, self.min_length, len(obj), location) 338 | 339 | if self.max_length is not None and len(obj) < self.max_length: 340 | raise ValidationException("Expected a list of at most length %d" % self.max_length, self.max_length, len(obj), location) 341 | 342 | result = [] 343 | for i in range(0, len(obj)): 344 | result.append(self.members(obj[i], "%s[%s]" % (location, i))) 345 | 346 | return result 347 | 348 | def __repr__(self): 349 | return "List(%s, min_length=%s, max_length=%s)" % (self.members, self.min_length, self.max_length) 350 | 351 | 352 | class Notify(Validator): 353 | 354 | """If a validator fails, provide a different message.""" 355 | 356 | def __init__(self, message, validator, expected=None, actual=None): 357 | self.message = message 358 | self.validator = validator 359 | self.expected = expected 360 | self.actual = actual 361 | 362 | def __call__(self, obj, location="/"): 363 | try: 364 | return self.validator(obj, location) 365 | 366 | except Exception as e: 367 | e.message = self.message 368 | e.expected = self.expected if self.expected is not None else e.expected 369 | e.actual = self.actual if self.actual is not None else e.actual 370 | raise e 371 | 372 | def __repr__(self): 373 | return repr(self.validator) 374 | 375 | 376 | class String(Validator): 377 | 378 | """Validate a string.""" 379 | 380 | def __init__(self, pattern=None, min_length=None, max_length=None, case_sensitive=True): 381 | self.pattern = pattern 382 | self.min_length = min_length 383 | self.max_length = max_length 384 | self.case_sensitive = case_sensitive 385 | self.pretty_pattern = pattern if isinstance(pattern, str) else pattern.pattern if pattern is not None else "" 386 | 387 | def __call__(self, obj, location="/"): 388 | if not isinstance(obj, str): 389 | raise ValidationException("Expected a string", str, type(obj), location) 390 | 391 | if self.pattern is not None: 392 | if isinstance(self.pattern, str): 393 | if self.case_sensitive: 394 | if self.pattern.casefold() != obj.casefold(): 395 | raise ValidationException("Expected '%s'" % self.pretty_pattern, self.pattern, obj, location) 396 | 397 | elif self.pattern != obj: 398 | raise ValidationException("Expected '%s'" % self.pretty_pattern, self.pattern, obj, location) 399 | 400 | else: 401 | if not self.pattern.match(obj): 402 | raise ValidationException("Expected '%s'" % self.pretty_pattern, self.pattern.pattern, obj, location) 403 | 404 | if self.min_length is not None and len(obj) < self.min_length: 405 | raise ValidationException("Expected a string of at least '%d' characters", self.min_length, len(obj), location) 406 | 407 | if self.max_length is not None and len(obj) > self.max_length: 408 | raise ValidationException("Expected a string of at most '%d' characters", self.max_length, len(obj), location) 409 | 410 | return obj if self.case_sensitive else CS(obj) 411 | 412 | def __repr__(self): 413 | return "String(pattern=%s, min_length=%s, max_length=%s, case_sensitive=%s)" % \ 414 | (self.pattern, self.min_length, self.max_length, self.case_sensitive) 415 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id$ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: Set up and install Giles. 20 | # 21 | ###################################################################### 22 | 23 | import os 24 | import sys 25 | from setuptools import setup 26 | 27 | from giles import get_release_string 28 | 29 | if sys.version_info < (3, 4, 0): 30 | print("Giles requires Python 3.4.0 or later.", file=sys.stderr) 31 | sys.exit(1) 32 | 33 | setup( 34 | name="giles", 35 | version=get_release_string(), 36 | install_requires=['Jinja2>=2.7.3', 'PyYAML>=3.11'], 37 | license="Affero GNU General Public License", 38 | author="Rob King", 39 | author_email="rking@korelogic.com", 40 | maintainer="Rob King", 41 | maintainer_email="giles-project@korelogic.com", 42 | description="Giles is a compiler for production systems.", 43 | long_description=open(os.path.join(os.path.dirname(__file__), 'README')).read(), 44 | url="http://www.korelogic.com", 45 | packages=['giles'], 46 | package_data={'giles': ['*.jinja']}, 47 | test_suite='tests.test_all', 48 | entry_points={ 49 | 'console_scripts': [ 50 | 'giles = giles.giles:main' 51 | ] 52 | } 53 | ) 54 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | ###################################################################### 4 | # 5 | # $Id$ 6 | # 7 | ###################################################################### 8 | # 9 | # Copyright 2011-2014 KoreLogic, Inc. All Rights Reserved. 10 | # 11 | # This software, having been partly or wholly developed and/or 12 | # sponsored by KoreLogic, Inc., is hereby released under the terms 13 | # and conditions set forth in the project's "README.LICENSE" file. 14 | # For a list of all contributors and sponsors, please refer to the 15 | # project's "README.CREDITS" file. 16 | # 17 | ###################################################################### 18 | # 19 | # Purpose: Run tests. 20 | # 21 | ###################################################################### 22 | 23 | """ 24 | __init__.py - run tests 25 | """ 26 | 27 | __author__ = "Rob King" 28 | __copyright__ = "Copyright (C) 2011-2013 KoreLogic, Inc. All Rights Reserved." 29 | __credits__ = [] 30 | __license__ = "See README.LICENSE" 31 | __version__ = "$Id$" 32 | __maintainer__ = "Rob King" 33 | __email__ = "rking@korelogic.com" 34 | __status__ = "Alpha" 35 | 36 | import doctest 37 | import glob 38 | import os 39 | import os.path 40 | import re 41 | import sqlite3 42 | import unittest 43 | 44 | from giles.giles import main 45 | from giles import caseless_string 46 | from giles import forbidden_names 47 | from giles import pyre 48 | from giles import validate 49 | 50 | 51 | class GilesCompilationTestCase(unittest.TestCase): 52 | 53 | def __init__(self, path): 54 | super().__init__() 55 | self.path = path 56 | self.output_path = "{0}.sql".format(self.path) 57 | 58 | def __str__(self): 59 | return "Compiling example engine {1}".format(str(self.__class__), self.path) 60 | 61 | def runTest(self): 62 | with self.assertRaises(SystemExit) as cm: 63 | main("-r", "-c", "-o", self.output_path, self.path) 64 | 65 | self.assertEqual(cm.exception.code, 0, "compilation failed") 66 | 67 | with open(self.output_path, "r") as schema_file: 68 | schema = schema_file.read() 69 | schema_file.close() 70 | self.assertTrue(len(schema) > 0, "compilation produced no output") 71 | 72 | db = sqlite3.connect(":memory:") 73 | db.create_function("regexp", 2, lambda x, y: re.search(x, y)) 74 | cursor = db.cursor() 75 | 76 | self.assertTrue(cursor.executescript(schema)) 77 | 78 | cursor.close() 79 | db.close() 80 | 81 | 82 | def test_all(): 83 | suite = unittest.TestSuite() 84 | suite.addTests(doctest.DocTestSuite(caseless_string)) 85 | suite.addTests(doctest.DocTestSuite(forbidden_names)) 86 | suite.addTests(doctest.DocTestSuite(pyre)) 87 | suite.addTests(doctest.DocTestSuite(validate)) 88 | 89 | for example in glob.glob(os.path.join(os.getcwd(), "examples", "*", "*.yml")): 90 | suite.addTest(GilesCompilationTestCase(example)) 91 | 92 | return suite 93 | 94 | if __name__ == '__main__': 95 | unittest.TextTestRunner(verbosity=2).run(test_all()) 96 | -------------------------------------------------------------------------------- /utils/version2string: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | ###################################################################### 3 | # 4 | # $Id: version2string,v 1.7 2013/02/14 01:34:38 mavrik Exp $ 5 | # 6 | ###################################################################### 7 | # 8 | # Copyright 2008-2013 The WebJob Project, All Rights Reserved. 9 | # 10 | ###################################################################### 11 | # 12 | # Purpose: Convert version numbers to a string representation. 13 | # 14 | ###################################################################### 15 | 16 | use strict; 17 | use File::Basename; 18 | use File::Path; 19 | use Getopt::Std; 20 | 21 | ###################################################################### 22 | # 23 | # Main Routine 24 | # 25 | ###################################################################### 26 | 27 | #################################################################### 28 | # 29 | # Punch in and go to work. 30 | # 31 | #################################################################### 32 | 33 | my ($sProgram); 34 | 35 | $sProgram = basename(__FILE__); 36 | 37 | #################################################################### 38 | # 39 | # Validation expressions. 40 | # 41 | #################################################################### 42 | 43 | my $sVersionRegex = qq(0x[0-9A-Fa-f]{8}); 44 | my $sTypeRegex = qq((?:cvs|program|tar)); 45 | 46 | #################################################################### 47 | # 48 | # Get Options. 49 | # 50 | #################################################################### 51 | 52 | my (%hOptions); 53 | 54 | if (!getopts('t:v:', \%hOptions)) 55 | { 56 | Usage($sProgram); 57 | } 58 | 59 | #################################################################### 60 | # 61 | # A type, '-t', is optional. 62 | # 63 | #################################################################### 64 | 65 | my $sType; 66 | 67 | $sType = (exists($hOptions{'t'})) ? $hOptions{'t'} : "program"; 68 | 69 | if (defined($sType) && $sType !~ /^$sTypeRegex$/) 70 | { 71 | print STDERR "$sProgram: Type='$sType' Error='Invalid version type.'\n"; 72 | exit(2); 73 | } 74 | 75 | #################################################################### 76 | # 77 | # A version, '-v', is required. 78 | # 79 | #################################################################### 80 | 81 | my $sVersion = (exists($hOptions{'v'})) ? $hOptions{'v'} : undef; 82 | 83 | if (!defined($sVersion)) 84 | { 85 | Usage($sProgram); 86 | } 87 | 88 | if ($sVersion !~ /^$sVersionRegex$/) 89 | { 90 | print STDERR "$sProgram: Version='$sVersion' Error='Invalid version.'\n"; 91 | exit(2); 92 | } 93 | 94 | #################################################################### 95 | # 96 | # If any arguments remain, it's an error. 97 | # 98 | #################################################################### 99 | 100 | if (scalar(@ARGV) > 0) 101 | { 102 | Usage($sProgram); 103 | } 104 | 105 | #################################################################### 106 | # 107 | # Do some work. 108 | # 109 | #################################################################### 110 | 111 | print VersionToString(hex($sVersion), $sType), "\n"; 112 | 113 | 1; 114 | 115 | ###################################################################### 116 | # 117 | # VersionToString 118 | # 119 | ###################################################################### 120 | 121 | sub VersionToString 122 | { 123 | my ($sVersion, $sType) = @_; 124 | 125 | my $sState = ($sVersion >> 10) & 0x03; 126 | my $sStateString = "xx"; 127 | if ($sState == 0) 128 | { 129 | $sStateString = "ds"; 130 | } 131 | elsif ($sState == 1) 132 | { 133 | $sStateString = "rc"; 134 | } 135 | elsif ($sState == 2) 136 | { 137 | $sStateString = "sr"; 138 | } 139 | elsif ($sState == 3) 140 | { 141 | $sStateString = "xs"; 142 | } 143 | 144 | my $sString = ""; 145 | if (($sVersion & 0xfff) == 0x800) 146 | { 147 | if ($sType =~ /^cvs$/) 148 | { 149 | $sString = sprintf 150 | ( 151 | "V%d_%d_%d", 152 | ($sVersion >> 28) & 0x0f, 153 | ($sVersion >> 20) & 0xff, 154 | ($sVersion >> 12) & 0xff 155 | ); 156 | } 157 | elsif ($sType =~ /^tar$/) 158 | { 159 | $sString = sprintf 160 | ( 161 | "%d.%d.%d", 162 | ($sVersion >> 28) & 0x0f, 163 | ($sVersion >> 20) & 0xff, 164 | ($sVersion >> 12) & 0xff 165 | ); 166 | } 167 | elsif ($sType =~ /^program$/) 168 | { 169 | $sString = sprintf 170 | ( 171 | "%d.%d.%d", 172 | ($sVersion >> 28) & 0x0f, 173 | ($sVersion >> 20) & 0xff, 174 | ($sVersion >> 12) & 0xff 175 | ); 176 | } 177 | } 178 | else 179 | { 180 | if ($sType =~ /^cvs$/) 181 | { 182 | $sString = sprintf 183 | ( 184 | "V%d_%d_%d_%s%d", 185 | ($sVersion >> 28) & 0x0f, 186 | ($sVersion >> 20) & 0xff, 187 | ($sVersion >> 12) & 0xff, 188 | uc($sStateString), 189 | $sVersion & 0x3ff 190 | ); 191 | } 192 | elsif ($sType =~ /^tar$/) 193 | { 194 | $sString = sprintf 195 | ( 196 | "%d.%d.%d.%s%d", 197 | ($sVersion >> 28) & 0x0f, 198 | ($sVersion >> 20) & 0xff, 199 | ($sVersion >> 12) & 0xff, 200 | $sStateString, 201 | $sVersion & 0x3ff 202 | ); 203 | } 204 | elsif ($sType =~ /^program$/) 205 | { 206 | $sString = sprintf 207 | ( 208 | "%d.%d.%d (%s%d)", 209 | ($sVersion >> 28) & 0x0f, 210 | ($sVersion >> 20) & 0xff, 211 | ($sVersion >> 12) & 0xff, 212 | $sStateString, 213 | $sVersion & 0x3ff 214 | ); 215 | } 216 | } 217 | 218 | return $sString; 219 | } 220 | 221 | 222 | ###################################################################### 223 | # 224 | # Usage 225 | # 226 | ###################################################################### 227 | 228 | sub Usage 229 | { 230 | my ($sProgram) = @_; 231 | print STDERR "\n"; 232 | print STDERR "Usage: $sProgram [-t {cvs|program|tar}] -v version\n"; 233 | print STDERR "\n"; 234 | exit(1); 235 | } 236 | -------------------------------------------------------------------------------- /utils/version_helper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | ###################################################################### 3 | # 4 | # $Id: version_helper,v 1.9 2013/02/14 01:20:34 mavrik Exp $ 5 | # 6 | ###################################################################### 7 | # 8 | # Copyright 2006-2013 The WebJob Project, All Rights Reserved. 9 | # 10 | ###################################################################### 11 | # 12 | # Purpose: Manage version numbers. 13 | # 14 | ###################################################################### 15 | 16 | use strict; 17 | use File::Basename; 18 | use File::Path; 19 | use Getopt::Std; 20 | 21 | ###################################################################### 22 | # 23 | # Main Routine 24 | # 25 | ###################################################################### 26 | 27 | #################################################################### 28 | # 29 | # Punch in and go to work. 30 | # 31 | #################################################################### 32 | 33 | my ($sProgram); 34 | 35 | $sProgram = basename(__FILE__); 36 | 37 | #################################################################### 38 | # 39 | # Validation expressions. 40 | # 41 | #################################################################### 42 | 43 | my $sBuildNumberRegex = qq((?:\\d+|[+])); 44 | my $sMajorNumberRegex = qq((?:\\d+|[+])); 45 | my $sMinorNumberRegex = qq((?:\\d+|[+])); 46 | my $sPatchNumberRegex = qq((?:\\d+|[+])); 47 | my $sStateNumberRegex = qq((?:[0-3+]|[dx]s|rc|sr)); 48 | 49 | #################################################################### 50 | # 51 | # Get Options. 52 | # 53 | #################################################################### 54 | 55 | my (%hOptions); 56 | 57 | if (!getopts('b:f:M:m:p:s:', \%hOptions)) 58 | { 59 | Usage($sProgram); 60 | } 61 | 62 | #################################################################### 63 | # 64 | # A filename is required, and can be '-' or a regular file. 65 | # 66 | #################################################################### 67 | 68 | my ($sFileHandle, $sFilename); 69 | 70 | if (!exists($hOptions{'f'})) 71 | { 72 | Usage($sProgram); 73 | } 74 | else 75 | { 76 | $sFilename = $hOptions{'f'}; 77 | if (!defined($sFilename) || length($sFilename) < 1) 78 | { 79 | Usage($sProgram); 80 | } 81 | if (-f $sFilename) 82 | { 83 | if (!open(FH, "< $sFilename")) 84 | { 85 | print STDERR "$sProgram: File='$sFilename' Error='$!'\n"; 86 | exit(2); 87 | } 88 | $sFileHandle = \*FH; 89 | } 90 | else 91 | { 92 | if ($sFilename ne '-') 93 | { 94 | print STDERR "$sProgram: File='$sFilename' Error='File must be regular.'\n"; 95 | exit(2); 96 | } 97 | $sFileHandle = \*STDIN; 98 | } 99 | } 100 | 101 | #################################################################### 102 | # 103 | # A MajorNumber, '-M', is optional. 104 | # 105 | #################################################################### 106 | 107 | my $sMajorNumber; 108 | 109 | $sMajorNumber = (exists($hOptions{'M'})) ? $hOptions{'M'} : undef; 110 | 111 | if (defined($sMajorNumber) && $sMajorNumber !~ /^$sMajorNumberRegex$/) 112 | { 113 | print STDERR "$sProgram: MajorNumber='$sMajorNumber' Error='Invalid major number.'\n"; 114 | exit(2); 115 | } 116 | 117 | #################################################################### 118 | # 119 | # A MinorNumber, '-m', is optional. 120 | # 121 | #################################################################### 122 | 123 | my $sMinorNumber; 124 | 125 | $sMinorNumber = (exists($hOptions{'m'})) ? $hOptions{'m'} : undef; 126 | 127 | if (defined($sMinorNumber) && $sMinorNumber !~ /^$sMinorNumberRegex$/) 128 | { 129 | print STDERR "$sProgram: MinorNumber='$sMinorNumber' Error='Invalid minor number.'\n"; 130 | exit(2); 131 | } 132 | 133 | #################################################################### 134 | # 135 | # An PatchNumber, '-p', is optional. 136 | # 137 | #################################################################### 138 | 139 | my $sPatchNumber; 140 | 141 | $sPatchNumber = (exists($hOptions{'p'})) ? $hOptions{'p'} : undef; 142 | 143 | if (defined($sPatchNumber) && $sPatchNumber !~ /^$sPatchNumberRegex$/) 144 | { 145 | print STDERR "$sProgram: PatchNumber='$sPatchNumber' Error='Invalid patch number.'\n"; 146 | exit(2); 147 | } 148 | 149 | #################################################################### 150 | # 151 | # A StateNumber, '-s', is optional. 152 | # 153 | #################################################################### 154 | 155 | my $sStateNumber; 156 | 157 | $sStateNumber = (exists($hOptions{'s'})) ? $hOptions{'s'} : undef; 158 | 159 | if (defined($sStateNumber) && $sStateNumber !~ /^$sStateNumberRegex$/) 160 | { 161 | print STDERR "$sProgram: StateNumber='$sStateNumber' Error='Invalid state number.'\n"; 162 | exit(2); 163 | } 164 | if (defined($sStateNumber) && $sStateNumber eq "ds") 165 | { 166 | $sStateNumber = 0; 167 | } 168 | elsif (defined($sStateNumber) && $sStateNumber eq "rc") 169 | { 170 | $sStateNumber = 1; 171 | } 172 | elsif (defined($sStateNumber) && $sStateNumber eq "sr") 173 | { 174 | $sStateNumber = 2; 175 | } 176 | elsif (defined($sStateNumber) && $sStateNumber eq "xs") 177 | { 178 | $sStateNumber = 3; 179 | } 180 | 181 | #################################################################### 182 | # 183 | # A BuildNumber, '-b', is optional. 184 | # 185 | #################################################################### 186 | 187 | my $sBuildNumber; 188 | 189 | $sBuildNumber = (exists($hOptions{'b'})) ? $hOptions{'b'} : undef; 190 | 191 | if (defined($sBuildNumber) && $sBuildNumber !~ /^$sBuildNumberRegex$/) 192 | { 193 | print STDERR "$sProgram: BuildNumber='$sBuildNumber' Error='Invalid build number.'\n"; 194 | exit(2); 195 | } 196 | 197 | #################################################################### 198 | # 199 | # If any arguments remain, it's an error. 200 | # 201 | #################################################################### 202 | 203 | if (scalar(@ARGV) > 0) 204 | { 205 | Usage($sProgram); 206 | } 207 | 208 | #################################################################### 209 | # 210 | # Attempt to locate/identify the current version number. 211 | # 212 | #################################################################### 213 | 214 | my ($sOldVersion, $sVersionFmt); 215 | 216 | while (my $sLine = <$sFileHandle>) 217 | { 218 | if ($sLine =~ /^#define VERSION (0x[0-9A-Fa-f]{8})\s*/) 219 | { 220 | $sOldVersion = hex($1); 221 | $sVersionFmt = "define"; 222 | last; 223 | } 224 | elsif ($sLine =~ /^\s*(0x[0-9A-Fa-f]{8})\s*$/) 225 | { 226 | $sOldVersion = hex($1); 227 | $sVersionFmt = "string"; 228 | last; 229 | } 230 | elsif ($sLine =~ /^\s*(?:version\s+=\s+)?(0x[0-9A-Fa-f]{8})\s*$/) 231 | { 232 | $sOldVersion = hex($1); 233 | $sVersionFmt = "assign"; 234 | last; 235 | } 236 | else 237 | { 238 | next; 239 | } 240 | } 241 | close($sFileHandle); 242 | 243 | if (!defined($sOldVersion)) 244 | { 245 | print STDERR "$sProgram: Error='Failed to locate/identify current version number.'\n"; 246 | exit(2); 247 | } 248 | 249 | if (!defined($sVersionFmt)) 250 | { 251 | print STDERR "$sProgram: Error='Failed to determine version format.'\n"; 252 | exit(2); 253 | } 254 | 255 | #################################################################### 256 | # 257 | # Compute the new version number. 258 | # 259 | #################################################################### 260 | 261 | my ($sNewVersion); 262 | 263 | $sNewVersion = $sOldVersion; 264 | 265 | if (defined($sMajorNumber)) 266 | { 267 | if ($sMajorNumber =~ /^\+$/) 268 | { 269 | $sNewVersion += 0x10000000; 270 | $sNewVersion &= 0xf0000000; 271 | } 272 | else 273 | { 274 | if ($sMajorNumber < 0 || $sMajorNumber > 15) 275 | { 276 | print STDERR "$sProgram: MajorNumber='$sMajorNumber' Error='Invalid major number.'\n"; 277 | exit(2); 278 | } 279 | $sNewVersion = (($sMajorNumber & 0xf) << 28) + ($sNewVersion & 0x0fffffff); 280 | } 281 | } 282 | 283 | if (defined($sMinorNumber)) 284 | { 285 | if ($sMinorNumber =~ /^\+$/) 286 | { 287 | $sNewVersion += 0x00100000; 288 | $sNewVersion &= 0xfff00000; 289 | } 290 | else 291 | { 292 | if ($sMinorNumber < 0 || $sMinorNumber > 255) 293 | { 294 | print STDERR "$sProgram: MinorNumber='$sMinorNumber' Error='Invalid minor number.'\n"; 295 | exit(2); 296 | } 297 | $sNewVersion = (($sMinorNumber & 0xff) << 20) + ($sNewVersion & 0xf00fffff); 298 | } 299 | } 300 | 301 | if (defined($sPatchNumber)) 302 | { 303 | if ($sPatchNumber =~ /^\+$/) 304 | { 305 | $sNewVersion += 0x00001000; 306 | $sNewVersion &= 0xfffff000; 307 | } 308 | else 309 | { 310 | if ($sPatchNumber < 0 || $sPatchNumber > 255) 311 | { 312 | print STDERR "$sProgram: PatchNumber='$sPatchNumber' Error='Invalid patch number.'\n"; 313 | exit(2); 314 | } 315 | $sNewVersion = (($sPatchNumber & 0xff) << 12) + ($sNewVersion & 0xfff00fff); 316 | } 317 | } 318 | 319 | if (defined($sStateNumber)) 320 | { 321 | if ($sStateNumber =~ /^\+$/) 322 | { 323 | $sNewVersion += 0x00000400; 324 | $sNewVersion &= 0xfffffc00; 325 | } 326 | else 327 | { 328 | if ($sStateNumber < 0 || $sStateNumber > 255) 329 | { 330 | print STDERR "$sProgram: StateNumber='$sStateNumber' Error='Invalid state number.'\n"; 331 | exit(2); 332 | } 333 | $sNewVersion = (($sStateNumber & 0x3) << 10) + ($sNewVersion & 0xfffff3ff); 334 | } 335 | } 336 | 337 | if (defined($sBuildNumber)) 338 | { 339 | if ($sBuildNumber =~ /^\+$/) 340 | { 341 | $sNewVersion += 0x00000001; 342 | } 343 | else 344 | { 345 | if ($sBuildNumber < 0 || $sBuildNumber > 255) 346 | { 347 | print STDERR "$sProgram: BuildNumber='$sBuildNumber' Error='Invalid build number.'\n"; 348 | exit(2); 349 | } 350 | $sNewVersion = ($sBuildNumber & 0x3ff) + ($sNewVersion & 0xfffffc00); 351 | } 352 | } 353 | 354 | #################################################################### 355 | # 356 | # Generate update/commit/tag commands the user can run manually. 357 | # 358 | #################################################################### 359 | 360 | my $sOldVersionString = VersionToString($sOldVersion, "tar"); 361 | my $sNewVersionString = VersionToString($sNewVersion, "tar"); 362 | my $so = sprintf("0x%08x", $sOldVersion); 363 | my $sn = sprintf("0x%08x", $sNewVersion); 364 | my $sCommand = "perl -p -i.bak "; 365 | if ($sVersionFmt eq "macro") 366 | { 367 | $sCommand .= " -e 's/define VERSION $so/define VERSION $sn/g;' $sFilename"; 368 | } 369 | else 370 | { 371 | $sCommand .= " -e 's/$so/$sn/g;' $sFilename"; 372 | } 373 | print $sCommand, "\n"; 374 | $sCommand = "cvs commit -m \"Updated version number ($sOldVersionString --> $sNewVersionString).\""; 375 | print $sCommand, "\n"; 376 | $sCommand = "cvs tag " . VersionToString($sNewVersion, "vcs"); 377 | print $sCommand, "\n"; 378 | if (((($sNewVersion >> 10) & 0x03) == 2) && (($sNewVersion & 0x3ff) == 0)) 379 | { 380 | $sCommand = "cvs tag " . VersionToString($sNewVersion, "vcs_sr0"); 381 | print $sCommand, "\n"; 382 | } 383 | 384 | 1; 385 | 386 | ###################################################################### 387 | # 388 | # VersionToString 389 | # 390 | ###################################################################### 391 | 392 | sub VersionToString 393 | { 394 | my ($sVersion, $sType) = @_; 395 | 396 | my $sState = ($sVersion >> 10) & 0x03; 397 | my $sStateString = "xx"; 398 | if ($sState == 0) 399 | { 400 | $sStateString = "ds"; 401 | } 402 | elsif ($sState == 1) 403 | { 404 | $sStateString = "rc"; 405 | } 406 | elsif ($sState == 2) 407 | { 408 | $sStateString = "sr"; 409 | } 410 | elsif ($sState == 3) 411 | { 412 | $sStateString = "xs"; 413 | } 414 | 415 | my $sString = ""; 416 | if ($sType =~ /^vcs$/) 417 | { 418 | $sString = sprintf 419 | ( 420 | "V%d_%d_%d_%s%d", 421 | ($sVersion >> 28) & 0x0f, 422 | ($sVersion >> 20) & 0xff, 423 | ($sVersion >> 12) & 0xff, 424 | uc($sStateString), 425 | $sVersion & 0x3ff 426 | ); 427 | } 428 | elsif ($sType =~ /^vcs_sr0$/) 429 | { 430 | $sString = sprintf 431 | ( 432 | "V%d_%d_%d", 433 | ($sVersion >> 28) & 0x0f, 434 | ($sVersion >> 20) & 0xff, 435 | ($sVersion >> 12) & 0xff 436 | ); 437 | } 438 | elsif ($sType =~ /^tar$/) 439 | { 440 | $sString = sprintf 441 | ( 442 | "%d.%d.%d.%s%d", 443 | ($sVersion >> 28) & 0x0f, 444 | ($sVersion >> 20) & 0xff, 445 | ($sVersion >> 12) & 0xff, 446 | $sStateString, 447 | $sVersion & 0x3ff 448 | ); 449 | } 450 | elsif ($sType =~ /^program$/) 451 | { 452 | $sString = sprintf 453 | ( 454 | "%d.%d.%d (%s%d)", 455 | ($sVersion >> 28) & 0x0f, 456 | ($sVersion >> 20) & 0xff, 457 | ($sVersion >> 12) & 0xff, 458 | $sStateString, 459 | $sVersion & 0x3ff 460 | ); 461 | } 462 | 463 | return $sString; 464 | } 465 | 466 | 467 | ###################################################################### 468 | # 469 | # Usage 470 | # 471 | ###################################################################### 472 | 473 | sub Usage 474 | { 475 | my ($sProgram) = @_; 476 | print STDERR "\n"; 477 | print STDERR "Usage: $sProgram [-M major] [-m minor] [-p patch] [-s state] [-b build] -f {file|-}\n"; 478 | print STDERR "\n"; 479 | exit(1); 480 | } 481 | --------------------------------------------------------------------------------