├── locks.log ├── .env ├── logo.png ├── test ├── rest-interface │ ├── .htaccess │ ├── composer.json │ ├── config │ │ └── config.json │ ├── vagrant │ │ └── bootstrap.sh │ ├── Vagrantfile │ └── utils │ │ ├── start-repset.sh │ │ └── create-repset.sh ├── unit │ ├── mongo │ │ ├── data │ │ │ ├── relatedContent.json │ │ │ ├── dates.json │ │ │ ├── view.json │ │ │ └── configQueueOperations.json │ │ ├── TestJobBase.php │ │ ├── TestConfigGenerator.php │ │ ├── ResqueJobTestBase.php │ │ ├── JobBaseTest.php │ │ ├── TriplesUtilTest.php │ │ ├── DateUtilTest.php │ │ ├── ConfigGeneratorTest.php │ │ └── MongoTripodDocumentStructureTest.php │ └── TimerTest.php ├── README.md ├── bootstrap.php └── performance │ └── mongo │ ├── MongoTripodPerformanceTestBase.php │ ├── MongoTripodConfigTest.php │ └── LargeGraphTest.php ├── src ├── exceptions │ ├── Exception.class.php │ ├── TimerException.class.php │ ├── ConfigException.class.php │ ├── CardinalityException.class.php │ ├── JobException.class.php │ ├── ViewException.class.php │ ├── SearchException.class.php │ └── LabellerException.class.php ├── ITripodConfigSerializer.php ├── mongo │ ├── util │ │ ├── DateUtil.class.php │ │ └── IndexUtils.class.php │ ├── IComposite.php │ ├── documents │ │ └── Tables.php │ ├── JobGroup.php │ ├── jobs │ │ ├── EnsureIndexes.class.php │ │ ├── ApplyOperation.class.php │ │ └── DiscoverImpactedSubjects.class.php │ ├── Labeller.class.php │ ├── providers │ │ └── ISearchProvider.php │ ├── ImpactedSubject.class.php │ ├── MongoTripodConstants.php │ ├── serializers │ │ └── NQuadSerializer.class.php │ ├── delegates │ │ └── TransactionLog.class.php │ └── base │ │ └── CompositeBase.class.php ├── ITripodConfig.php ├── TripodStatFactory.class.php ├── ITripodStat.php ├── TripodConfigFactory.php ├── IEventHook.php ├── Config.php ├── classes │ ├── Timer.class.php │ └── StatsD.class.php ├── IDriver.php └── tripod.inc.php ├── .gitignore ├── .editorconfig ├── phpunit.xml ├── docker-compose.clusters.yml ├── scripts └── mongo │ ├── worker.inc.php │ ├── startApplyWorkers.sh │ ├── startDiscoverWorkers.sh │ ├── startWorkers.sh │ ├── common.inc.php │ ├── BSONToQuads.php │ ├── ensureIndexes.php │ ├── validateConfig.php │ ├── triplesToBSON.php │ ├── BSONToTriples.php │ ├── loadTriples.php │ ├── discoverUnnamespacedUris.php │ ├── createViews.php │ ├── createTables.php │ ├── createSearchDocuments.php │ └── detectNamespaces.php ├── docker ├── Dockerfile-php73 └── Dockerfile-php74 ├── docker-compose.yml ├── LICENSE ├── composer.json ├── docs ├── primers │ ├── hooks.md │ └── views.md └── operations.md └── .circleci └── config.yml /locks.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | RESQUE_SERVER=redis -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/techfromsage/tripod-php/HEAD/logo.png -------------------------------------------------------------------------------- /test/rest-interface/.htaccess: -------------------------------------------------------------------------------- 1 | RewriteEngine On 2 | RewriteCond %{REQUEST_FILENAME} !-f 3 | RewriteRule ^ index.php [QSA,L] 4 | -------------------------------------------------------------------------------- /src/exceptions/Exception.class.php: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | test/performance 10 | 11 | 12 | test/unit 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /test/rest-interface/config/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "statsConfig": { 3 | "class":"Tripod\StatsD", 4 | "config": { 5 | "host" : "localhost", 6 | "port" : 8125 7 | } 8 | }, 9 | "tripod" : { 10 | "async" : { 11 | "generate_views":false, 12 | "generate_table_rows" : true, 13 | "generate_search_index_docs" : true 14 | }, 15 | "readPreference":"primaryPreferred" 16 | }, 17 | "read-repeat": 0 18 | } -------------------------------------------------------------------------------- /src/ITripodConfigSerializer.php: -------------------------------------------------------------------------------- 1 | pushHandler(new \Monolog\Handler\StreamHandler('php://stderr', Psr\Log\LogLevel::WARNING)); // resque too chatty on NOTICE & INFO. YMMV 7 | 8 | // this is so tripod itself uses the same logger 9 | \Tripod\Mongo\DriverBase::$logger = new \Monolog\Logger("TRIPOD-JOB",array(new \Monolog\Handler\StreamHandler('php://stderr', Psr\Log\LogLevel::DEBUG))); -------------------------------------------------------------------------------- /docker/Dockerfile-php73: -------------------------------------------------------------------------------- 1 | FROM php:7.3.33-cli 2 | 3 | RUN apt-get update && apt-get install -y --no-install-recommends \ 4 | ca-certificates \ 5 | curl \ 6 | git \ 7 | unzip \ 8 | zip \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | RUN curl -sLo /tmp/mongosh.deb https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.15_amd64.deb \ 12 | && dpkg -i /tmp/mongosh.deb \ 13 | && rm /tmp/mongosh.deb 14 | 15 | COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/ 16 | COPY --from=composer:2.7.7 /usr/bin/composer /usr/local/bin/ 17 | 18 | RUN install-php-extensions pcntl mongodb-1.6.1 19 | -------------------------------------------------------------------------------- /docker/Dockerfile-php74: -------------------------------------------------------------------------------- 1 | FROM php:7.4.33-cli 2 | 3 | RUN apt-get update && apt-get install -y --no-install-recommends \ 4 | ca-certificates \ 5 | curl \ 6 | git \ 7 | unzip \ 8 | zip \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | RUN curl -sLo /tmp/mongosh.deb https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.15_amd64.deb \ 12 | && dpkg -i /tmp/mongosh.deb \ 13 | && rm /tmp/mongosh.deb 14 | 15 | COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/ 16 | COPY --from=composer:2.7.7 /usr/bin/composer /usr/local/bin/ 17 | 18 | RUN install-php-extensions pcntl mongodb-1.19.3 19 | -------------------------------------------------------------------------------- /src/mongo/util/DateUtil.class.php: -------------------------------------------------------------------------------- 1 | target = $target; 19 | parent::__construct("Could not label: $target"); 20 | } 21 | 22 | /** 23 | * @return string 24 | */ 25 | public function getTarget() 26 | { 27 | return $this->target; 28 | } 29 | } -------------------------------------------------------------------------------- /test/unit/mongo/TestConfigGenerator.php: -------------------------------------------------------------------------------- 1 | get_class($this), 'filename' => $this->fileName]; 12 | } 13 | 14 | public static function deserialize(array $config) 15 | { 16 | $instance = new self(); 17 | $instance->fileName = $config['filename']; 18 | $cfg = json_decode(file_get_contents($config['filename']), true); 19 | $instance->loadConfig($cfg); 20 | return $instance; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/ITripodConfig.php: -------------------------------------------------------------------------------- 1 | getMockBuilder(Resque_Job::class) 10 | ->onlyMethods(['getInstance', 'getArguments']) 11 | ->setConstructorArgs(['test', get_class($job), $job->args]) 12 | ->getMock(); 13 | $mockJob->expects($this->atLeastOnce()) 14 | ->method('getInstance') 15 | ->will($this->returnValue($job)); 16 | $mockJob->expects($this->any()) 17 | ->method('getArguments') 18 | ->will($this->returnValue($job->args)); 19 | $mockJob->perform(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/ITripodStat.php: -------------------------------------------------------------------------------- 1 | pushHandler(new Monolog\Handler\NullHandler()); 20 | Tripod\Mongo\DriverBase::$logger = $log; 21 | -------------------------------------------------------------------------------- /scripts/mongo/common.inc.php: -------------------------------------------------------------------------------- 1 | args = $this->getArgs(); 9 | $job->job = new Resque_Job('queue', ['id' => uniqid()]); 10 | 11 | $this->assertInstanceOf(Tripod\Mongo\IConfigInstance::class, $job->getTripodConfig()); 12 | } 13 | 14 | protected function getArgs() 15 | { 16 | return [ 17 | 'tripodConfig' => Tripod\Config::getConfig(), 18 | 'storeName' => 'tripod_php_testing', 19 | 'podName' => 'CBD_testing', 20 | 'changes' => ['http://example.com/resources/foo' => ['rdf:type', 'dct:title']], 21 | 'operations' => [OP_VIEWS, OP_TABLES, OP_SEARCH], 22 | 'contextAlias' => 'http://talisaspire.com/', 23 | ]; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /scripts/mongo/BSONToQuads.php: -------------------------------------------------------------------------------- 1 | -c > bsondata.txt \n"; 11 | die(); 12 | } 13 | 14 | array_shift($argv); 15 | $config = json_decode(file_get_contents($argv[0]), true); 16 | \Tripod\Config::setConfig($config); 17 | 18 | $tu = new \Tripod\Mongo\TriplesUtil(); 19 | 20 | while (($line = fgets(STDIN)) !== false) { 21 | $line = rtrim($line); 22 | $doc = json_decode($line, true); 23 | $context = $doc['_id']['c']; 24 | 25 | $graph = new \Tripod\Mongo\MongoGraph(); 26 | $graph->add_tripod_array($doc); 27 | 28 | echo $graph->to_nquads($context); 29 | } 30 | ?> -------------------------------------------------------------------------------- /test/performance/mongo/MongoTripodPerformanceTestBase.php: -------------------------------------------------------------------------------- 1 | /etc/php5/apache2/conf.d/mongodb.ini 12 | sed -i "s/DocumentRoot \/var\/www/DocumentRoot \/vagrant/" /etc/apache2/sites-enabled/000-default 13 | sed -i "s/Directory \/var\/www\//Directory \/vagrant\//" /etc/apache2/sites-enabled/000-default 14 | sed -i "s/AllowOverride None/AllowOverride All/" /etc/apache2/sites-enabled/000-default 15 | service apache2 restart 16 | -------------------------------------------------------------------------------- /src/mongo/IComposite.php: -------------------------------------------------------------------------------- 1 | toTableRow($input)); 10 | } 11 | /** 12 | * Sets the array value to the modeled table row value 13 | * 14 | * @param array $data DB document array 15 | * @return void 16 | */ 17 | public function bsonUnserialize(array $data) 18 | { 19 | $this->exchangeArray($this->toTableRow($data)); 20 | } 21 | 22 | /** 23 | * Models the table row from the source data 24 | * 25 | * @param array $doc Database document 26 | * @return array 27 | */ 28 | protected function toTableRow(array $doc) 29 | { 30 | $result = isset($doc['value']) ? $doc['value'] : []; 31 | if (isset($result[_IMPACT_INDEX])) { 32 | unset($result[_IMPACT_INDEX]); 33 | } 34 | return $result; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /test/rest-interface/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | 7 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 8 | # All Vagrant configuration is done here. The most common configuration 9 | # options are documented and commented below. For a complete reference, 10 | # please see the online documentation at vagrantup.com. 11 | 12 | # Every Vagrant virtual environment requires a box to build off of. 13 | config.vm.box = "https://cloud-images.ubuntu.com/vagrant/precise/current/precise-server-cloudimg-amd64-vagrant-disk1.box" 14 | config.vm.provision :shell, :path => "vagrant/bootstrap.sh" 15 | 16 | config.vm.network "private_network", ip: "#{ENV['VAGRANT_IP_TRIPOD_TESTING']}" 17 | config.vm.hostname = "tripod-performance-testing.vagrant" 18 | config.vm.network "forwarded_port", guest: 27017, host: 27717 19 | config.vm.provider "virtualbox" do |v| 20 | v.memory = 2048 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /scripts/mongo/ensureIndexes.php: -------------------------------------------------------------------------------- 1 | setMongoCursorTimeout(-1); 20 | 21 | $ei = new \Tripod\Mongo\Jobs\EnsureIndexes(); 22 | 23 | $t = new \Tripod\Timer(); 24 | $t->start(); 25 | print("About to start scheduling indexing jobs for $storeName...\n"); 26 | $ei->createJob($storeName, $forceReindex, $background); 27 | $t->stop(); 28 | print "Finished scheduling ensure indexes jobs, took {$t->result()} seconds\n"; 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Talis Group Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "talis/tripod-php", 3 | "type": "library", 4 | "description": "A library for managing RDF data in Mongo", 5 | "keywords": ["rdf","sparql"], 6 | "homepage": "https://github.com/talis/tripod-php", 7 | "license": "MIT", 8 | "authors": [ 9 | { 10 | "name": "Chris Clarke", 11 | "email": "cc@talis.com", 12 | "homepage": "http://talis.com/" 13 | }, 14 | { 15 | "name": "Nadeem Shabir", 16 | "email": "ns@talis.com", 17 | "homepage": "http://talis.com/" 18 | } 19 | ], 20 | "suggest": { 21 | "resque/php-resque": "Redis backed library for background jobs" 22 | }, 23 | "require": { 24 | "php" : ">=7.3", 25 | "mongodb/mongodb": "*", 26 | "monolog/monolog" : "~1.13", 27 | "semsol/arc2": "2.2.6" 28 | }, 29 | "require-dev": { 30 | "phpunit/phpunit": "^9.6.20", 31 | "resque/php-resque": "v1.3.6", 32 | "squizlabs/php_codesniffer": "3.2.*" 33 | }, 34 | "autoload": { 35 | "classmap": ["src/"] 36 | }, 37 | "autoload-dev": { 38 | "classmap": ["test/"] 39 | }, 40 | "scripts": { 41 | "test": "phpunit" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /scripts/mongo/validateConfig.php: -------------------------------------------------------------------------------- 1 | getMessage() . "\n"; 48 | } 49 | -------------------------------------------------------------------------------- /test/unit/mongo/data/dates.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": 4 | { 5 | "r":"baseData:foo1234", 6 | "c":"http://talisaspire.com/" 7 | }, 8 | "rdf:type": 9 | [ 10 | { 11 | "u":"bibo:Document" 12 | } 13 | ], 14 | "dct:title" : { 15 | "l" : "A document title" 16 | }, 17 | "dct:isVersionOf" : { 18 | "u" : "http://talisaspire.com/works/4d101f63c10a6" 19 | }, 20 | "dct:updated" : { 21 | "l" : "2017-02-23T14:49:02+00:00" 22 | }, 23 | "dct:published" : { 24 | "l" : "2017-02-22T14:49:02+00:00" 25 | } 26 | }, 27 | { 28 | "_id": 29 | { 30 | "r":"baseData:foo12345", 31 | "c":"http://talisaspire.com/" 32 | }, 33 | "rdf:type": 34 | [ 35 | { 36 | "u":"bibo:Document" 37 | } 38 | ], 39 | "dct:title" : { 40 | "l" : "A document title" 41 | }, 42 | "dct:isVersionOf" : { 43 | "u" : "http://talisaspire.com/works/4d101f63c10a6" 44 | }, 45 | "dct:updated" : { 46 | "l" : "2017-02-23T14:49:02+00:00" 47 | }, 48 | "dct:published" : { 49 | "l" : "2017-02-23T14:49:02+00:00" 50 | } 51 | } 52 | ] -------------------------------------------------------------------------------- /scripts/mongo/triplesToBSON.php: -------------------------------------------------------------------------------- 1 | <'); 27 | 28 | if (empty($currentSubject)) // set for first iteration 29 | { 30 | $currentSubject = $subject; 31 | } 32 | 33 | if ($currentSubject!=$subject) // once subject changes, we have all triples for that subject, flush to Mongo 34 | { 35 | print(json_encode($tu->getTArrayAbout($currentSubject,$triples,\Tripod\Config::getInstance()->getDefaultContextAlias()))."\n"); 36 | $currentSubject=$subject; // reset current subject to next subject 37 | $triples = array(); // reset triples 38 | } 39 | 40 | $triples[] = $line; 41 | } 42 | 43 | // last doc 44 | print(json_encode($tu->getTArrayAbout($currentSubject,$triples,\Tripod\Config::getInstance()->getDefaultContextAlias()))."\n"); 45 | 46 | ?> 47 | 48 | -------------------------------------------------------------------------------- /test/unit/mongo/TriplesUtilTest.php: -------------------------------------------------------------------------------- 1 | . '; 11 | $triples[] = ' . '; 12 | $triples[] = ' . '; 13 | $triples[] = ' "1548-774X" . '; 14 | 15 | $expectedDoc = [ 16 | '_id' => ['r' => 'http://serials.talisaspire.com/issn/0893-0465', 'c' => 'http://talisaspire.com/'], 17 | 'foaf:page' => [ 18 | [ 19 | 'u' => 'http://www.ingentaconnect.com/content/bpl/ciso'], 20 | [ 21 | 'u' => 'http://onlinelibrary.wiley.com/journal/10.1111/(ISSN)1548-744X'], 22 | ], 23 | 'rdf:type' => [ 24 | 'u' => 'bibo:Journal', 25 | ], 26 | 'bibo:eissn' => [ 27 | 'l' => '1548-774X', 28 | ], 29 | ]; 30 | $this->assertEquals($expectedDoc, $tu->getTArrayAbout('http://serials.talisaspire.com/issn/0893-0465', $triples, 'http://talisaspire.com/')); 31 | } 32 | 33 | // todo: add triples test 34 | } 35 | -------------------------------------------------------------------------------- /scripts/mongo/BSONToTriples.php: -------------------------------------------------------------------------------- 1 | -c > bsondata.txt \n"; 10 | die(); 11 | } 12 | array_shift($argv); 13 | $config = json_decode(file_get_contents($argv[0]), true); 14 | \Tripod\Config::setConfig($config); 15 | 16 | $tu = new \Tripod\Mongo\TriplesUtil(); 17 | 18 | while (($line = fgets(STDIN)) !== false) { 19 | $line = rtrim($line); 20 | 21 | $graph = new \Tripod\Mongo\MongoGraph(); 22 | $doc = json_decode($line, true); 23 | 24 | if(array_key_exists("_id", $doc)) { 25 | 26 | $subject = $doc['_id']; 27 | 28 | unset($doc["_id"]); 29 | if( array_key_exists("_version", $doc)) { 30 | unset($doc["_version"]); 31 | } 32 | 33 | foreach($doc as $property=>$values) { 34 | if(isset($values['value'])) { 35 | $doc[$property] = array($values); 36 | } 37 | } 38 | 39 | foreach($doc as $property=>$values) { 40 | foreach($values as $value) { 41 | if($value['type'] == "literal" ) { 42 | $graph->add_literal_triple($subject, $graph->qname_to_uri($property), $value['value']); 43 | } else { 44 | $graph->add_resource_triple($subject, $graph->qname_to_uri($property), $value['value']); 45 | } 46 | } 47 | } 48 | 49 | print($graph->to_ntriples()); 50 | } 51 | } 52 | ?> -------------------------------------------------------------------------------- /docs/primers/hooks.md: -------------------------------------------------------------------------------- 1 | Event Hooks 2 | === 3 | 4 | Tripod has the concept of Event Hooks for you to be able to hook in custom code which will be executed when certain events are triggered. 5 | 6 | Your custom code should be packacked in the form of a class that extends `\Tripod\IEventHook`. This means it will implement three methods: 7 | 8 | * `pre` - will be executed just before the event occurs 9 | * `success` - will be executed if the event is successful 10 | * `failure` - will be executed if the event was attempted, but deemed to fail 11 | 12 | These methods are each passed an `$args` array, the contents of which depend on the event type 13 | 14 | Hooks cannot influence the execution flow by throwing exceptions - any exceptions are logged but not propagated. 15 | 16 | Register a hook by calling `IDriver::registerHook` passing the event type constant as the first arg and an instance of your class as the second 17 | 18 | Type: Save Changes 19 | --- 20 | 21 | `\Tripod\IEventHook::EVENT_SAVE_CHANGES` 22 | 23 | This event is triggered when `\Tripod\IDriver::saveChanges()` is called. 24 | 25 | Tripod will call `pre()`/`failure()/success()` with the following arg key values: 26 | 27 | 1. `pod` The pod that is being saved to (`string`) 28 | 1. `oldGraph` The old graph state being presented for save (`\Tripod\ExtendedGraph`) 29 | 1. `newGraph` The new graph state being presented for save (`\Tripod\ExtendedGraph`) 30 | 1. `context` The named graph being saved into (`string`) 31 | 32 | There are some additional values added for `success()`: 33 | 34 | 1. `changeSet` The complete changeset applied (`\ITripod\ChangeSet`) 35 | 1. `subjectsAndPredicatesOfChange` Subjects and predicates actually updated, keyed by subject (`array`) 36 | 1. `transaction_id` The resulting transaction ID in the tlog (`string`) 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/IEventHook.php: -------------------------------------------------------------------------------- 1 | ] [-n ] [-p ] [-l " 1>&2; exit 0; } 21 | 22 | while getopts r:n:p:l: opt; do 23 | case "${opt}" in 24 | r) 25 | REPSET_NUMBER=${OPTARG} 26 | REPSET_NAME="rs" 27 | REPSET_NAME+=$REPSET_NUMBER 28 | : $((BASE_PORT_MODIFIER=$REPSET_NUMBER * 100)) 29 | : $((BASE_PORT=$BASE_PORT + $BASE_PORT_MODIFIER)) 30 | ;; 31 | n) 32 | NODE_COUNT=${OPTARG} 33 | ;; 34 | p) 35 | DB_BASE_PATH=${OPTARG} 36 | ;; 37 | l) 38 | LOG_PATH=${OPTARG} 39 | ;; 40 | *) 41 | usage 42 | ;; 43 | esac 44 | done 45 | 46 | COUNTER=0; 47 | while [ $COUNTER -lt $NODE_COUNT ]; do 48 | : $((MONGO_PORT=$BASE_PORT+$COUNTER)) 49 | CURR_PATH=$DB_BASE_PATH/rs$REPSET_NUMBER-$COUNTER 50 | echo "Creating $CURR_PATH, if it does not exist" 51 | mkdir -p $CURR_PATH 52 | echo "Starting Mongo node $COUNTER on port $MONGO_PORT" 53 | nohup mongod --port $MONGO_PORT --dbpath $CURR_PATH --replSet $REPSET_NAME --smallfiles --oplogSize 128 >> $LOG_PATH/mongod-$REPSET_NAME-$COUNTER.log & 54 | : $((COUNTER=$COUNTER+1)) 55 | done 56 | -------------------------------------------------------------------------------- /src/Config.php: -------------------------------------------------------------------------------- 1 | array(OP_VIEWS=>false, OP_TABLES=>true, OP_SEARCH=>true)); 15 | 16 | ``` 17 | 18 | By default (if no OP_ASYNC array is passed), views will be regenerated synchronously and table rows and search documents 19 | will be generated via a background job using php-resque. 20 | 21 | Queues 22 | ------ 23 | 24 | Tripod divides composite regeneration into two jobs: DiscoverImpactedSubjects (which isolates the resources affected by 25 | the changes) and ApplyOperation (which actually invalidates the composite documents and regenerates the new ones). These can 26 | be run either synchronously or asynchronously. 27 | 28 | Queues are configured via environment variables, although defaults will be set if no environment variables are found. 29 | 30 | ` RESQUE_SERVER ` 31 | defines the Redis backend for Resque (default: localhost:6379) 32 | 33 | ` APP_ENV ` 34 | (Optional) - the queues will be namespaced to the particular environment (e.g. ` tripod:::production::discover `) 35 | 36 | ` TRIPOD_DISCOVER_QUEUE ` 37 | The queue name for DiscoverImpactedSubjects jobs (if ` $APP_ENV ` is set, defaults to ` tripod::$APP_ENV::discover `, otherwise ` tripod::discover `) 38 | 39 | ` TRIPOD_APPLY_QUEUE ` 40 | The queue name for ApplyOperations jobs (if ` $APP_ENV ` is set, defaults to ` tripod::$APP_ENV::apply `, otherwise ` tripod::apply `) 41 | 42 | There are some generic php-resque job worker scripts in scripts/mongo to get you started. 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /test/unit/mongo/DateUtilTest.php: -------------------------------------------------------------------------------- 1 | getMongoDate(); 9 | 10 | $_id = [ 11 | 'r' => 'http://talisaspire.com/resources/testEtag' . microtime(false), 12 | 'c' => 'http://talisaspire.com/']; 13 | $doc = [ 14 | '_id' => $_id, 15 | 'dct:title' => ['l' => 'etag'], 16 | '_version' => 0, 17 | '_cts' => $updatedAt, 18 | '_uts' => $updatedAt, 19 | ]; 20 | $config->getCollectionForCBD( 21 | 'tripod_php_testing', 22 | 'CBD_testing' 23 | )->insertOne($doc, ['w' => 1]); 24 | 25 | $date = Tripod\Mongo\DateUtil::getMongoDate(); 26 | 27 | $this->assertInstanceOf(MongoDB\BSON\UTCDateTime::class, $date); 28 | $this->assertEquals(13, strlen($date->__toString())); 29 | } 30 | 31 | public function testGetMongoDateWithParam() 32 | { 33 | $config = Tripod\Config::getInstance(); 34 | $updatedAt = (new Tripod\Mongo\DateUtil())->getMongoDate(); 35 | 36 | $_id = [ 37 | 'r' => 'http://talisaspire.com/resources/testEtag' . microtime(false), 38 | 'c' => 'http://talisaspire.com/']; 39 | $doc = [ 40 | '_id' => $_id, 41 | 'dct:title' => ['l' => 'etag'], 42 | '_version' => 0, 43 | '_cts' => $updatedAt, 44 | '_uts' => $updatedAt, 45 | ]; 46 | $config->getCollectionForCBD( 47 | 'tripod_php_testing', 48 | 'CBD_testing' 49 | )->insertOne($doc, ['w' => 1]); 50 | 51 | $time = floor(microtime(true) * 1000); 52 | $date = Tripod\Mongo\DateUtil::getMongoDate($time); 53 | 54 | $this->assertInstanceOf(MongoDB\BSON\UTCDateTime::class, $date); 55 | $this->assertEquals(13, strlen($date->__toString())); 56 | $this->assertEquals($time, $date->__toString()); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /test/unit/mongo/data/view.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id" : { 3 | "r" : "http://example.com/resources/1", 4 | "c" : "http://talisaspire.com/", 5 | "type" : "v_bib_resource_full" 6 | }, 7 | "value" : { 8 | "_graphs" : [ 9 | { 10 | "_id" : { 11 | "r" : "http://example.com/resources/1/authors", 12 | "c" : "http://talisaspire.com/" 13 | }, 14 | "rdf:_1" : { 15 | "u" : "http://example.com/people/1" 16 | }, 17 | "rdf:type" : { 18 | "u" : "rdf:Seq" 19 | } 20 | }, 21 | { 22 | "_id" : { 23 | "r" : "http://example.com/people/1", 24 | "c" : "http://talisaspire.com/" 25 | }, 26 | "rdf:type" : { 27 | "u" : "foaf:Person" 28 | }, 29 | "foaf:name" : { 30 | "l" : "Miller, Daniel" 31 | } 32 | }, 33 | { 34 | "_id" : { 35 | "r" : "http://example.com/organisations/1", 36 | "c" : "http://talisaspire.com/" 37 | }, 38 | "rdf:type" : { 39 | "u" : "foaf:Organization" 40 | }, 41 | "foaf:name" : { 42 | "l" : "Routledge" 43 | } 44 | }, 45 | { 46 | "_id" : { 47 | "r" : "http://example.com/resources/1", 48 | "c" : "http://talisaspire.com/" 49 | }, 50 | "dct:date" : { 51 | "l" : "2003" 52 | }, 53 | "dct:publisher" : { 54 | "u" : "http://example.com/organisations/1" 55 | }, 56 | "dct:source" : { 57 | "u" : "http://example.com/catalog/542114" 58 | }, 59 | "dct:title" : { 60 | "l" : "Material cultures: why some things matter" 61 | }, 62 | "bibo:authorList" : { 63 | "u" : "http://example.com/resources/1/authors" 64 | }, 65 | "bibo:isbn10" : { 66 | "l" : "1857286863" 67 | }, 68 | "bibo:volume" : { 69 | "l" : "Consumption and space" 70 | }, 71 | "rdf:type" : { 72 | "u" : "bibo:Book" 73 | } 74 | } 75 | ], 76 | "_impactIndex" : [ 77 | { 78 | "r" : "http://example.com/resources/1", 79 | "c" : "http://talisaspire.com/" 80 | }, 81 | { 82 | "r" : "http://example.com/resources/1/authors", 83 | "c" : "http://talisaspire.com/" 84 | }, 85 | { 86 | "r" : "http://example.com/people/1", 87 | "c" : "http://talisaspire.com/" 88 | }, 89 | { 90 | "r" : "http://example.com/organisations/1", 91 | "c" : "http://talisaspire.com/" 92 | } 93 | ] 94 | } 95 | } -------------------------------------------------------------------------------- /test/unit/TimerTest.php: -------------------------------------------------------------------------------- 1 | %s\n", get_class($this), $this->getName()); 11 | } 12 | 13 | /** START: result() tests */ 14 | public function testResultWhenStartTimeNotSet() 15 | { 16 | $timer = new Timer(); 17 | $this->expectException(Exception::class); 18 | $this->expectExceptionMessage('Timer: start method not called !'); 19 | $timer->result(); 20 | } 21 | 22 | public function testResultWhenEndTimeNotSet() 23 | { 24 | $timer = new Timer(); 25 | $timer->start(); 26 | $this->expectException(Exception::class); 27 | $this->expectExceptionMessage('Timer: stop method not called !'); 28 | $timer->result(); 29 | } 30 | 31 | public function testResultGetTimeInMilliSeconds() 32 | { 33 | $timer = new Timer(); 34 | $timer->start(); 35 | sleep(1); // Let's pause for one seconds otherwise we will get 0 as a result. 36 | $timer->stop(); 37 | $status = ($timer->result() >= 1000) ? true : false; 38 | $this->assertTrue($status); 39 | } 40 | /** END: result() tests */ 41 | 42 | /** START: microResult() tests */ 43 | public function testMicroResultWhenStartTimeNotSet() 44 | { 45 | $timer = new Timer(); 46 | $this->expectException(Exception::class); 47 | $this->expectExceptionMessage('Timer: start method not called !'); 48 | $timer->result(); 49 | } 50 | 51 | public function testMicroResultWhenEndTimeNotSet() 52 | { 53 | $timer = new Timer(); 54 | $timer->start(); 55 | $this->expectException(Exception::class); 56 | $this->expectExceptionMessage('Timer: stop method not called !'); 57 | $timer->result(); 58 | } 59 | 60 | public function testMicroResultGetTimeInMilliSeconds() 61 | { 62 | $timer = new Timer(); 63 | $timer->start(); 64 | sleep(1); // Let's pause for one seconds otherwise we might get 0 as a result. 65 | $timer->stop(); 66 | $status = ($timer->microResult() >= 1000000) ? true : false; 67 | $this->assertTrue($status); 68 | } 69 | /* END: microResult() tests */ 70 | } 71 | -------------------------------------------------------------------------------- /test/performance/mongo/MongoTripodConfigTest.php: -------------------------------------------------------------------------------- 1 | config = json_decode(file_get_contents(dirname(__FILE__) . '/../../unit/mongo/data/config.json'), true); 34 | } 35 | 36 | /** 37 | * Post test completion actions. 38 | */ 39 | protected function tearDown(): void 40 | { 41 | $this->config = []; 42 | parent::tearDown(); 43 | } 44 | 45 | /** 46 | * Note: Current version of this test tried to create 1000 objects within 6000ms which is reasonable at this time. 47 | * Any change to this class if make it a more a big number it should be validated and tested to ensure performance impact. 48 | * 49 | * Create some instances of Config to see what amount of time is taken in creating instance and processing in constructor. 50 | */ 51 | public function testCreateMongoTripodConfigObject() 52 | { 53 | $testStartTime = microtime(); 54 | 55 | // Let's try to create 1000 objects to see how much time they take. 56 | for ($i = 0; $i < self::BENCHMARK_OBJECT_CREATE_ITERATIONS; $i++) { 57 | Tripod\Config::setConfig($this->config); 58 | $instance = Tripod\Config::getInstance(); 59 | } 60 | 61 | $testEndTime = microtime(); 62 | $this->assertLessThan( 63 | self::BENCHMARK_OBJECT_CREATE_TIME, 64 | $this->getTimeDifference($testStartTime, $testEndTime), 65 | 'It should always take less than ' . self::BENCHMARK_OBJECT_CREATE_TIME . 'ms to create ' . self::BENCHMARK_OBJECT_CREATE_ITERATIONS . ' objects of Config class' 66 | ); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /scripts/mongo/loadTriples.php: -------------------------------------------------------------------------------- 1 | loadTriplesAbout($subject,$triples,$storeName,$podName); 19 | } 20 | catch (Exception $e) 21 | { 22 | print "Exception for subject $subject failed with message: ".$e->getMessage()."\n"; 23 | $errors[] = $subject; 24 | } 25 | } 26 | 27 | $timer = new \Tripod\Timer(); 28 | $timer->start(); 29 | 30 | if ($argc!=4) 31 | { 32 | echo "usage: ./loadTriples.php storename podname tripodConfig.json < ntriplesdata\n"; 33 | die(); 34 | } 35 | array_shift($argv); 36 | 37 | $storeName = $argv[0]; 38 | $podName = $argv[1]; 39 | \Tripod\Config::setConfig(json_decode(file_get_contents($argv[2]),true)); 40 | 41 | $i=0; 42 | $currentSubject = ""; 43 | $triples = array(); 44 | $errors = array(); // array of subjects that failed to insert, even after retry... 45 | $loader = new \Tripod\Mongo\TriplesUtil(); 46 | 47 | while (($line = fgets(STDIN)) !== false) { 48 | $i++; 49 | 50 | if (($i % 250000)==0) 51 | { 52 | print "Memory: ".memory_get_usage()."\n"; 53 | } 54 | 55 | $line = rtrim($line); 56 | $parts = preg_split("/\s/",$line); 57 | $subject = trim($parts[0],'><'); 58 | 59 | 60 | if (empty($currentSubject)) // set for first iteration 61 | { 62 | $currentSubject = $subject; 63 | } 64 | else if ($currentSubject!=$subject) // once subject changes, we have all triples for that subject, flush to Mongo 65 | { 66 | load($loader,$currentSubject,$triples,$errors,$podName,$storeName); 67 | $currentSubject=$subject; // reset current subject to next subject 68 | $triples = array(); // reset triples 69 | } 70 | $triples[] = $line; 71 | } 72 | 73 | // last doc 74 | load($loader,$currentSubject,$triples,$errors,$podName,$storeName); 75 | 76 | $timer->stop(); 77 | print "This script ran in ".$timer->result()." milliseconds\n"; 78 | 79 | echo "Processed ".($i)." triples"; 80 | if (count($errors)>0) 81 | { 82 | echo "Insert errors on ".count($errors)." subjects\n"; 83 | var_dump($errors); //todo: decide what to do with errors... 84 | } 85 | -------------------------------------------------------------------------------- /src/mongo/JobGroup.php: -------------------------------------------------------------------------------- 1 | storeName = $storeName; 21 | if (!$groupId) { 22 | $groupId = new ObjectId(); 23 | } elseif (!$groupId instanceof ObjectId) { 24 | $groupId = new ObjectId($groupId); 25 | } 26 | $this->id = $groupId; 27 | } 28 | 29 | /** 30 | * Update the number of jobs 31 | * 32 | * @param integer $count Number of jobs in group 33 | * @return void 34 | */ 35 | public function setJobCount($count) 36 | { 37 | $this->getMongoCollection()->updateOne( 38 | ['_id' => $this->getId()], 39 | ['$set' => ['count' => $count]], 40 | ['upsert' => true] 41 | ); 42 | } 43 | 44 | /** 45 | * Update the number of jobs by $inc. To decrement, use a negative integer 46 | * 47 | * @param integer $inc Number to increment or decrement by 48 | * @return integer Updated job count 49 | */ 50 | public function incrementJobCount($inc = 1) 51 | { 52 | $updateResult = $this->getMongoCollection()->findOneAndUpdate( 53 | ['_id' => $this->getId()], 54 | ['$inc' => ['count' => $inc]], 55 | ['upsert' => true, 'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER] 56 | ); 57 | if (\is_array($updateResult)) { 58 | return $updateResult['count']; 59 | } elseif (isset($updateResult->count)) { 60 | return $updateResult->count; 61 | } 62 | } 63 | 64 | /** 65 | * @return ObjectId 66 | */ 67 | public function getId() 68 | { 69 | return $this->id; 70 | } 71 | 72 | /** 73 | * For mocking 74 | * 75 | * @return \MongoDB\Collection 76 | */ 77 | protected function getMongoCollection() 78 | { 79 | if (!isset($this->collection)) { 80 | $config = \Tripod\Config::getInstance(); 81 | 82 | $this->collection = $config->getCollectionForJobGroups($this->storeName); 83 | } 84 | return $this->collection; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /test/rest-interface/utils/create-repset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # This script creates and starts a very basic Mongo replica set on the localhost 5 | # 6 | # Usage: create-repset.sh [-r repset-number] [-n number-of-nodes] [-p base-path-to-use-for-databases] 7 | # 8 | # -r The repset number is for running multiple repsets concurrently, the number is used for generating the repset 9 | # repset name ("rs3" for -r 3) and port (27017 + (repset number x 100)): defaults to 0 10 | # -n Number of nodes to start in repset: defaults to 2 11 | # -p Database path: defaults to /tmp/mongodb 12 | # -l Path to log: defaults to /tmp 13 | 14 | BASE_PORT=27017; 15 | DB_BASE_PATH="/tmp/mongodb" 16 | NODE_COUNT=2 17 | REPSET_NUMBER=0 18 | REPSET_NAME="rs$REPSET_NUMBER" 19 | LOG_PATH="/tmp" 20 | 21 | usage() { echo "Usage: $0 [-r ] [-n ] [-p ] [-l " 1>&2; exit 0; } 22 | 23 | while getopts r:n:p:l: opt; do 24 | case "${opt}" in 25 | r) 26 | REPSET_NUMBER=${OPTARG} 27 | REPSET_NAME="rs" 28 | REPSET_NAME+=$REPSET_NUMBER 29 | : $((BASE_PORT_MODIFIER=$REPSET_NUMBER * 100)) 30 | : $((BASE_PORT=$BASE_PORT + $BASE_PORT_MODIFIER)) 31 | ;; 32 | n) 33 | NODE_COUNT=${OPTARG} 34 | ;; 35 | p) 36 | DB_BASE_PATH=${OPTARG} 37 | ;; 38 | l) 39 | LOG_PATH=${OPTARG} 40 | ;; 41 | *) 42 | usage 43 | ;; 44 | esac 45 | done 46 | REPSET_CONF="/tmp/create-repset-$REPSET_NAME.js" 47 | ./start-repset.sh -r $REPSET_NUMBER -n $NODE_COUNT -p $DB_BASE_PATH -l $LOG_PATH 48 | 49 | cat > $REPSET_CONF < $REPSET_NODE_SCRIPT "" 74 | COUNTER=1; 75 | while [ $COUNTER -lt $NODE_COUNT ]; do 76 | : $((MONGO_PORT=$BASE_PORT+$COUNTER)) 77 | : $((COUNTER=$COUNTER+1)) 78 | cat >> $REPSET_NODE_SCRIPT <debugLog('Ensuring indexes for tenant=' . $this->args[self::STORENAME_KEY]. ', reindex=' . $this->args[self::REINDEX_KEY] . ', background=' . $this->args[self::BACKGROUND_KEY]); 26 | 27 | $this->getIndexUtils()->ensureIndexes( 28 | $this->args[self::REINDEX_KEY], 29 | $this->args[self::STORENAME_KEY], 30 | $this->args[self::BACKGROUND_KEY] 31 | ); 32 | } 33 | 34 | /** 35 | * Stat string for successful job timer 36 | * 37 | * @return string 38 | */ 39 | protected function getStatTimerSuccessKey() 40 | { 41 | return MONGO_QUEUE_ENSURE_INDEXES_SUCCESS; 42 | } 43 | 44 | /** 45 | * Stat string for failed job increment 46 | * 47 | * @return string 48 | */ 49 | protected function getStatFailureIncrementKey() 50 | { 51 | return MONGO_QUEUE_ENSURE_INDEXES_FAIL; 52 | } 53 | 54 | /** 55 | * This method is use to schedule an EnsureIndexes job. 56 | * 57 | * @param string $storeName 58 | * @param booelan $reindex 59 | * @param string $queueName 60 | */ 61 | public function createJob($storeName, $reindex, $background, $queueName = null) 62 | { 63 | $configInstance = $this->getConfigInstance(); 64 | if (!$queueName) { 65 | $queueName = $configInstance::getEnsureIndexesQueueName(); 66 | } elseif (strpos($queueName, $configInstance::getEnsureIndexesQueueName()) === false) { 67 | $queueName = $configInstance::getEnsureIndexesQueueName() . '::' . $queueName; 68 | } 69 | 70 | $data = [ 71 | self::STORENAME_KEY => $storeName, 72 | self::REINDEX_KEY => $reindex, 73 | self::BACKGROUND_KEY => $background 74 | ]; 75 | 76 | $this->submitJob($queueName, get_class($this), array_merge($data, $this->generateConfigJobArgs())); 77 | } 78 | 79 | /** 80 | * @return \Tripod\Mongo\IndexUtils 81 | */ 82 | protected function getIndexUtils() 83 | { 84 | return new \Tripod\Mongo\IndexUtils(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /test/performance/mongo/LargeGraphTest.php: -------------------------------------------------------------------------------- 1 | tripod = new Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', ['defaultContext' => 'http://talisaspire.com/']); 27 | 28 | $this->loadLargeGraphData(); 29 | } 30 | 31 | protected function loadLargeGraphData() 32 | { 33 | $docs = json_decode(file_get_contents(dirname(__FILE__) . '/data/largeGraph.json'), true); 34 | foreach ($docs as $d) { 35 | $this->addDocument($d); 36 | } 37 | } 38 | 39 | protected function getConfigLocation() 40 | { 41 | return dirname(__FILE__) . '/../../unit/mongo/data/config.json'; 42 | } 43 | 44 | public function testUpdateSingleTripleOfLargeGraph() 45 | { 46 | $uri = 'http://largegraph/1'; 47 | 48 | $testStartTime = microtime(); 49 | 50 | $graph = new Tripod\ExtendedGraph(); 51 | $graph->add_literal_triple($uri, 'http://rdfs.org/sioc/spec/name', 'new name'); 52 | $this->tripod->saveChanges(new Tripod\ExtendedGraph(), $graph); 53 | 54 | $testEndTime = microtime(); 55 | 56 | $this->assertLessThan( 57 | self::BENCHMARK_SAVE_TIME, 58 | $this->getTimeDifference($testStartTime, $testEndTime), 59 | 'It should always take less than ' . self::BENCHMARK_SAVE_TIME . 'ms to save a triple to a large graph' 60 | ); 61 | } 62 | 63 | public function testDescribeOfLargeGraph() 64 | { 65 | $uri = 'http://largegraph/1'; 66 | 67 | $testStartTime = microtime(); 68 | 69 | $graph = new Tripod\ExtendedGraph(); 70 | $graph->add_literal_triple($uri, 'http://rdfs.org/sioc/spec/name', 'new name'); 71 | $this->tripod->describeResource($uri); 72 | 73 | $testEndTime = microtime(); 74 | 75 | $this->assertLessThan( 76 | self::BENCHMARK_DESCRIBE_TIME, 77 | $this->getTimeDifference($testStartTime, $testEndTime), 78 | 'It should always take less than ' . self::BENCHMARK_DESCRIBE_TIME . 'ms to describe large graph' 79 | ); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/mongo/Labeller.class.php: -------------------------------------------------------------------------------- 1 | _ns = array( 18 | 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', 19 | 'rdfs' => 'http://www.w3.org/2000/01/rdf-schema#', 20 | 'owl' => 'http://www.w3.org/2002/07/owl#', 21 | 'cs' => 'http://purl.org/vocab/changeset/schema#', 22 | ); 23 | $config = \Tripod\Config::getInstance(); 24 | $ns = $config->getNamespaces(); 25 | foreach ($ns as $prefix=>$uri) 26 | { 27 | $this->set_namespace_mapping($prefix,$uri); 28 | } 29 | } 30 | 31 | /** 32 | * If labeller can generate a qname for this uri, it will return it. Otherwise just returns the original uri 33 | * @param $uri 34 | * @return string 35 | */ 36 | function uri_to_alias($uri) { 37 | try 38 | { 39 | $retVal = $this->uri_to_qname($uri); 40 | } 41 | catch (\Tripod\Exceptions\LabellerException $e) {} 42 | return (empty($retVal)) ? $uri : $retVal; 43 | } 44 | 45 | /** 46 | * If labeller can generate a uri for this qname, it will return it. Otherwise just returns the original qname 47 | * @param $qname 48 | * @return string 49 | */ 50 | function qname_to_alias($qname) { 51 | try 52 | { 53 | $retVal = $this->qname_to_uri($qname); 54 | } 55 | catch (\Tripod\Exceptions\LabellerException $e) {} 56 | return (empty($retVal)) ? $qname : $retVal; 57 | } 58 | 59 | /** 60 | * @param string $qName 61 | * @return string 62 | * @throws \Tripod\Exceptions\LabellerException 63 | */ 64 | public function qname_to_uri($qName) 65 | { 66 | $retVal = parent::qname_to_uri($qName); 67 | if (empty($retVal)) throw new \Tripod\Exceptions\LabellerException($qName); 68 | return $retVal; 69 | } 70 | 71 | 72 | // overrides the default behaviour of trying to return a ns even if the prefix is not registered - instead, throw exception 73 | /** 74 | * @param string $ns 75 | * @return string 76 | * @throws \Tripod\Exceptions\LabellerException 77 | */ 78 | public function get_prefix($ns) { 79 | $prefix = array_search($ns, $this->_ns); 80 | if ( $prefix != null && $prefix !== FALSE) { 81 | return $prefix; 82 | } 83 | else 84 | { 85 | throw new \Tripod\Exceptions\LabellerException($ns); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/mongo/providers/ISearchProvider.php: -------------------------------------------------------------------------------- 1 | start_time = $this->getMicrotime(); 37 | } 38 | 39 | /** 40 | * Captures current microtime as end time. Call this as soon as event execution is complete. 41 | */ 42 | public function stop() 43 | { 44 | $this->end_time = $this->getMicrotime(); 45 | } 46 | 47 | /** 48 | * Calculate difference between start and end time of event and return in milli-seconds. 49 | * @throws \Exception 50 | * @return number time difference in milliseconds between start and end time of event 51 | */ 52 | public function result() 53 | { 54 | if (is_null($this->start_time)) 55 | { 56 | throw new \Exception('Timer: start method not called !'); 57 | } 58 | else if (is_null($this->end_time)) 59 | { 60 | throw new \Exception('Timer: stop method not called !'); 61 | } 62 | 63 | if ($this->result==null) 64 | { 65 | list($endTimeMicroSeconds, $endTimeSeconds) = explode(' ', $this->end_time); 66 | list($startTimeMicroSeconds, $startTimeSeconds) = explode(' ', $this->start_time); 67 | 68 | $differenceInMilliSeconds = ((float)$endTimeSeconds - (float)$startTimeSeconds)*1000; 69 | 70 | $this->result = round(($differenceInMilliSeconds + ((float)$endTimeMicroSeconds *1000)) - (float)$startTimeMicroSeconds *1000); 71 | } 72 | return $this->result; 73 | } 74 | 75 | /** 76 | * Calculate difference between start and end time of event and return in micro-seconds. 77 | * @return number time difference in micro seconds between start and end time of event 78 | * @throws \Exception, if either of or both of start or stop method are not called before this method 79 | */ 80 | public function microResult() 81 | { 82 | if (is_null($this->start_time)) 83 | { 84 | throw new \Tripod\Exceptions\TimerException('Timer: start method not called !'); 85 | } 86 | else if (is_null($this->end_time)) 87 | { 88 | throw new \Tripod\Exceptions\TimerException('Timer: stop method not called !'); 89 | } 90 | 91 | if ($this->micro_result==null) 92 | { 93 | list($endTimeMicroSeconds, $endTimeSeconds) = explode(' ', $this->end_time); 94 | list($startTimeMicroSeconds, $startTimeSeconds) = explode(' ', $this->start_time); 95 | 96 | $differenceInMicroSeconds = ((float)$endTimeSeconds - (float)$startTimeSeconds)*1000000; 97 | 98 | $this->micro_result = round(($differenceInMicroSeconds + ((float)$endTimeMicroSeconds *1000000)) - (float)$startTimeMicroSeconds *1000000); 99 | } 100 | return $this->micro_result; 101 | } 102 | 103 | /** 104 | * @return string current system time in pair of seconds microseconds. 105 | */ 106 | private function getMicrotime() 107 | { 108 | return microtime(); 109 | } 110 | } -------------------------------------------------------------------------------- /scripts/mongo/discoverUnnamespacedUris.php: -------------------------------------------------------------------------------- 1 | ['root' => 'array', 'document' => 'array', 'array' => 'array']] 19 | ); 20 | 21 | /** @var \MongoDB\Database $db */ 22 | $db = $client->selectDatabase($argv[1]); 23 | 24 | /** 25 | * @param string $uri 26 | * @param string|null $baseUri 27 | * @return bool 28 | */ 29 | function isUnNamespaced($uri,$baseUri=null) 30 | { 31 | if ($baseUri==null) 32 | { 33 | return (strpos($uri,'http://')===0 || strpos($uri,'https://')===0); 34 | } 35 | else 36 | { 37 | return strpos($uri,$baseUri)===0; 38 | } 39 | } 40 | 41 | $results = array(); 42 | foreach ($db->listCollections() as $collectionInfo) 43 | { 44 | 45 | /** @var \MongoDB\Collection $collection*/ 46 | if (strpos($collectionInfo->getName(),'CBD_')===0) // only process CBD_collections 47 | { 48 | $collection = $db->selectCollection($collectionInfo->getName()); 49 | echo "Checking out {$collectionInfo->getName()}\n"; 50 | $count = 0; 51 | foreach ($collection->find() as $doc) 52 | { 53 | if (!isset($doc['_id']) || !isset($doc['_id']['r'])) 54 | { 55 | echo " Illegal doc: no _id or missing _id.r"; 56 | } 57 | else 58 | { 59 | if (isUnNamespaced($doc['_id']['r'], (isset($argv[2]) ? $argv[2] : null) )) 60 | { 61 | echo " Un-namespaced subject: {$doc['_id']['r']}\n"; 62 | $count++; 63 | } 64 | } 65 | foreach ($doc as $property=>$value) 66 | { 67 | if (strpos($property,"_")===0) // ignore meta fields, _id, _version, _uts etc. 68 | { 69 | continue; 70 | } 71 | else 72 | { 73 | if (isset($value['l'])) 74 | { 75 | // ignore, is a literal 76 | continue; 77 | } 78 | else if (isset($value['u'])) 79 | { 80 | if (isUnNamespaced($value['u'], (isset($argv[2]) ? $argv[2] : null))) 81 | { 82 | echo " Un-namespaced object uri (single value): {$value['u']}\n"; 83 | $count++; 84 | } 85 | } 86 | else 87 | { 88 | foreach ($value as $v) 89 | { 90 | if (isset($v['u'])) 91 | { 92 | if (isUnNamespaced($v['u'], (isset($argv[2]) ? $argv[2] : null))) 93 | { 94 | echo " Un-namespaced object uri (multiple value): {$v['u']}\n"; 95 | $count++; 96 | } 97 | } 98 | } 99 | } 100 | } 101 | } 102 | } 103 | $results[] = "{$collectionInfo->getName()} has $count un-namespaced uris"; 104 | echo "Done with {$collectionInfo->getName()}\n"; 105 | } 106 | } 107 | echo "\n".implode("\n",$results)."\n"; 108 | 109 | ?> -------------------------------------------------------------------------------- /src/mongo/ImpactedSubject.class.php: -------------------------------------------------------------------------------- 1 | resourceId = $resourceId; 61 | } 62 | 63 | if (in_array($operation,array(OP_VIEWS,OP_TABLES,OP_SEARCH))) 64 | { 65 | $this->operation = $operation; 66 | } 67 | else 68 | { 69 | throw new \Tripod\Exceptions\Exception("Invalid operation: $operation"); 70 | } 71 | 72 | $this->storeName = $storeName; 73 | $this->podName = $podName; 74 | $this->specTypes = $specTypes; 75 | 76 | if($stat) 77 | { 78 | $this->stat = $stat; 79 | } 80 | } 81 | 82 | /** 83 | * @return string 84 | */ 85 | public function getOperation() 86 | { 87 | return $this->operation; 88 | } 89 | 90 | /** 91 | * @return string 92 | */ 93 | public function getPodName() 94 | { 95 | return $this->podName; 96 | } 97 | 98 | /** 99 | * @return array 100 | */ 101 | public function getResourceId() 102 | { 103 | return $this->resourceId; 104 | } 105 | 106 | /** 107 | * @return array 108 | */ 109 | public function getSpecTypes() 110 | { 111 | return $this->specTypes; 112 | } 113 | 114 | /** 115 | * @return string 116 | */ 117 | public function getStoreName() 118 | { 119 | return $this->storeName; 120 | } 121 | 122 | /** 123 | * Serialises the data as an array 124 | * @return array 125 | */ 126 | public function toArray() 127 | { 128 | return array( 129 | "resourceId" => $this->resourceId, 130 | "operation" => $this->operation, 131 | "specTypes" => $this->specTypes, 132 | "storeName" => $this->storeName, 133 | "podName" => $this->podName 134 | ); 135 | } 136 | 137 | /** 138 | * Perform the update on the composite defined by the operation 139 | */ 140 | public function update() 141 | { 142 | $tripod = $this->getTripod(); 143 | if(isset($this->stat)) 144 | { 145 | $tripod->setStat($this->stat); 146 | } 147 | $tripod->getComposite($this->operation)->update($this); 148 | } 149 | 150 | /** 151 | * For mocking 152 | * @return \Tripod\Mongo\Driver 153 | */ 154 | protected function getTripod() 155 | { 156 | return new Driver($this->getPodName(),$this->getStoreName(),array( 157 | 'readPreference' => ReadPreference::RP_PRIMARY 158 | )); 159 | } 160 | } -------------------------------------------------------------------------------- /scripts/mongo/createViews.php: -------------------------------------------------------------------------------- 1 | getViewSpecification($storeName, $viewId); 63 | if (array_key_exists("from",$viewSpec)) 64 | { 65 | \Tripod\Config::getInstance()->setMongoCursorTimeout(-1); 66 | 67 | print "Generating $viewId"; 68 | $tripod = new \Tripod\Mongo\Driver($viewSpec['from'], $storeName, array('stat'=>$stat)); 69 | $views = $tripod->getTripodViews(); 70 | if ($id) 71 | { 72 | print " for $id....\n"; 73 | $views->generateView($viewId, $id, null, $queue); 74 | } 75 | else 76 | { 77 | print " for all views....\n"; 78 | $views->generateView($viewId, null, null, $queue); 79 | } 80 | } 81 | } 82 | 83 | $t = new \Tripod\Timer(); 84 | $t->start(); 85 | 86 | \Tripod\Config::setConfig(json_decode(file_get_contents($configLocation),true)); 87 | 88 | if(isset($options['s']) || isset($options['storename'])) 89 | { 90 | $storeName = isset($options['s']) ? $options['s'] : $options['storename']; 91 | } 92 | else 93 | { 94 | $storeName = null; 95 | } 96 | 97 | if(isset($options['v']) || isset($options['spec'])) 98 | { 99 | $viewId = isset($options['v']) ? $options['v'] : $options['spec']; 100 | } 101 | else 102 | { 103 | $viewId = null; 104 | } 105 | 106 | if(isset($options['i']) || isset($options['id'])) 107 | { 108 | $id = isset($options['i']) ? $options['i'] : $options['id']; 109 | } 110 | else 111 | { 112 | $id = null; 113 | } 114 | 115 | $queue = null; 116 | if(isset($options['a']) || isset($options['async'])) 117 | { 118 | if(isset($options['q']) || isset($options['queue'])) 119 | { 120 | $queue = $options['queue']; 121 | } 122 | else 123 | { 124 | $queue = \Tripod\Config::getInstance()->getApplyQueueName(); 125 | } 126 | } 127 | 128 | $stat = null; 129 | 130 | if(isset($options['stat-loader'])) 131 | { 132 | $stat = require_once $options['stat-loader']; 133 | } 134 | 135 | if ($viewId) 136 | { 137 | generateViews($id, $viewId, $storeName, $stat, $queue); 138 | } 139 | else 140 | { 141 | foreach(\Tripod\Config::getInstance()->getViewSpecifications($storeName) as $viewSpec) 142 | { 143 | generateViews($id, $viewSpec['_id'], $storeName, $stat, $queue); 144 | } 145 | } 146 | 147 | $t->stop(); 148 | print "Views created in ".$t->result()." secs\n"; -------------------------------------------------------------------------------- /scripts/mongo/createTables.php: -------------------------------------------------------------------------------- 1 | getTableSpecification($storeName, $tableId); 63 | if (array_key_exists("from",$tableSpec)) 64 | { 65 | \Tripod\Config::getInstance()->setMongoCursorTimeout(-1); 66 | 67 | print "Generating $tableId"; 68 | $tripod = new \Tripod\Mongo\Driver($tableSpec['from'], $storeName, array('stat'=>$stat)); 69 | $tTables = $tripod->getTripodTables(); 70 | if ($id) 71 | { 72 | print " for $id....\n"; 73 | $tTables->generateTableRows($tableId, $id); 74 | } 75 | else 76 | { 77 | print " for all tables....\n"; 78 | $tTables->generateTableRows($tableId, null, null, $queue); 79 | } 80 | } 81 | } 82 | 83 | $t = new \Tripod\Timer(); 84 | $t->start(); 85 | 86 | \Tripod\Config::setConfig(json_decode(file_get_contents($configLocation),true)); 87 | 88 | if(isset($options['s']) || isset($options['storename'])) 89 | { 90 | $storeName = isset($options['s']) ? $options['s'] : $options['storename']; 91 | } 92 | else 93 | { 94 | $storeName = null; 95 | } 96 | 97 | if(isset($options['t']) || isset($options['spec'])) 98 | { 99 | $tableId = isset($options['t']) ? $options['t'] : $options['spec']; 100 | } 101 | else 102 | { 103 | $tableId = null; 104 | } 105 | 106 | if(isset($options['i']) || isset($options['id'])) 107 | { 108 | $id = isset($options['i']) ? $options['i'] : $options['id']; 109 | } 110 | else 111 | { 112 | $id = null; 113 | } 114 | 115 | $queue = null; 116 | if(isset($options['a']) || isset($options['async'])) 117 | { 118 | if(isset($options['q']) || isset($options['queue'])) 119 | { 120 | $queue = $options['queue']; 121 | } 122 | else 123 | { 124 | $queue = \Tripod\Config::getInstance()->getApplyQueueName(); 125 | } 126 | } 127 | 128 | $stat = null; 129 | 130 | if(isset($options['stat-loader'])) 131 | { 132 | $stat = require_once $options['stat-loader']; 133 | } 134 | 135 | if ($tableId) 136 | { 137 | generateTables($id, $tableId, $storeName, $stat, $queue); 138 | } 139 | else 140 | { 141 | foreach(\Tripod\Config::getInstance()->getTableSpecifications($storeName) as $tableSpec) 142 | { 143 | generateTables($id, $tableSpec['_id'], $storeName, $stat, $queue); 144 | } 145 | } 146 | 147 | $t->stop(); 148 | print "Tables created in ".$t->result()." secs\n"; -------------------------------------------------------------------------------- /scripts/mongo/createSearchDocuments.php: -------------------------------------------------------------------------------- 1 | getSearchDocumentSpecification($storeName, $specId); 62 | if (array_key_exists("from",$spec)) 63 | { 64 | \Tripod\Config::getInstance()->setMongoCursorTimeout(-1); 65 | 66 | print "Generating $specId"; 67 | $tripod = new \Tripod\Mongo\Driver($spec['from'], $storeName, array('stat'=>$stat)); 68 | $search = $tripod->getSearchIndexer(); 69 | if ($id) 70 | { 71 | print " for $id....\n"; 72 | $search->generateSearchDocuments($specId, $id, null, $queue); 73 | } 74 | else 75 | { 76 | print " for all tables....\n"; 77 | $search->generateSearchDocuments($specId, null, null, $queue); 78 | } 79 | } 80 | } 81 | 82 | $t = new \Tripod\Timer(); 83 | $t->start(); 84 | 85 | \Tripod\Config::setConfig(json_decode(file_get_contents($configLocation),true)); 86 | 87 | if(isset($options['s']) || isset($options['storename'])) 88 | { 89 | $storeName = isset($options['s']) ? $options['s'] : $options['storename']; 90 | } 91 | else 92 | { 93 | $storeName = null; 94 | } 95 | 96 | if(isset($options['d']) || isset($options['spec'])) 97 | { 98 | $specId = isset($options['d']) ? $options['t'] : $options['spec']; 99 | } 100 | else 101 | { 102 | $specId = null; 103 | } 104 | 105 | if(isset($options['i']) || isset($options['id'])) 106 | { 107 | $id = isset($options['i']) ? $options['i'] : $options['id']; 108 | } 109 | else 110 | { 111 | $id = null; 112 | } 113 | 114 | $queue = null; 115 | if(isset($options['a']) || isset($options['async'])) 116 | { 117 | if(isset($options['q']) || isset($options['queue'])) 118 | { 119 | $queue = $options['queue']; 120 | } 121 | else 122 | { 123 | $queue = \Tripod\Config::getInstance()->getApplyQueueName(); 124 | } 125 | } 126 | 127 | $stat = null; 128 | 129 | if(isset($options['stat-loader'])) 130 | { 131 | $stat = require_once $options['stat-loader']; 132 | } 133 | 134 | if ($specId) 135 | { 136 | generateSearchDocuments($id, $specId, $storeName, $stat, $queue); 137 | } 138 | else 139 | { 140 | foreach(\Tripod\Config::getInstance()->getSearchDocumentSpecifications($storeName) as $searchSpec) 141 | { 142 | generateSearchDocuments($id, $searchSpec['_id'], $storeName, $stat, $queue); 143 | } 144 | } 145 | 146 | $t->stop(); 147 | print "Search documents created in ".$t->result()." secs\n"; -------------------------------------------------------------------------------- /src/mongo/MongoTripodConstants.php: -------------------------------------------------------------------------------- 1 | "foo", 16 | "collection"=>"bar", 17 | "connStr"=>"baz" 18 | ); 19 | $config = array( 20 | "namespaces"=>array(), 21 | "defaultContext"=>"http://example.com/", 22 | "transaction_log"=>$dummyDbConfig, 23 | "es_config"=>array( 24 | "endpoint"=>"http://example.com/", 25 | "indexes"=>array(), 26 | "search_document_specifications"=>array() 27 | ), 28 | "queue"=>$dummyDbConfig, 29 | "databases"=>array( 30 | "default"=>array( 31 | "connStr"=>"baz", 32 | "collections"=>array() 33 | ) 34 | ), 35 | ); 36 | 37 | \Tripod\Config::setConfig($config); 38 | 39 | 40 | $util = new \Tripod\Mongo\TriplesUtil(); 41 | $objectNs = array(); 42 | $i=0; 43 | while (($line = fgets(STDIN)) !== false) { 44 | $i++; 45 | 46 | $line = rtrim($line); 47 | $parts = preg_split("/\s/",$line); 48 | $subject = trim($parts[0],'><'); 49 | 50 | if (($i % 2500)==0) 51 | { 52 | print '.'; 53 | } 54 | if (($i % 50000)==0) 55 | { 56 | foreach ($objectNs as $key=>$val) 57 | { 58 | if ($val < 5) 59 | { 60 | // flush 61 | unset($objectNs[$key]); 62 | } 63 | } 64 | gc_collect_cycles(); 65 | print 'F'; 66 | } 67 | 68 | if (empty($currentSubject)) // set for first iteration 69 | { 70 | $currentSubject = $subject; 71 | } 72 | else if ($currentSubject!=$subject) // once subject changes, we have all triples for that subject, flush to Mongo 73 | { 74 | $ns = $util->extractMissingPredicateNs($triples); 75 | if (count($ns)>0) 76 | { 77 | $newNsConfig = array(); 78 | foreach($ns as $n) 79 | { 80 | $prefix = $util->suggestPrefix($n); 81 | if (array_key_exists($prefix,$config['namespaces'])) 82 | { 83 | $prefix = $prefix.uniqid(); 84 | } 85 | $newNsConfig[$prefix] = $n; 86 | echo "\nFound ns $n suggest prefix $prefix"; 87 | $config["namespaces"] = array_merge($config["namespaces"],$newNsConfig); 88 | \Tripod\Config::setConfig($config); 89 | } 90 | } 91 | $ns = $util->extractMissingObjectNs($triples); 92 | if (count($ns)>0) 93 | { 94 | $newNsConfig = array(); 95 | foreach($ns as $n) 96 | { 97 | if (array_key_exists($n,$objectNs)) { 98 | $objectNs[$n]++; 99 | } 100 | else 101 | { 102 | $objectNs[$n] = 1; 103 | } 104 | if ($objectNs[$n]>500) 105 | { 106 | $prefix = $util->suggestPrefix($n); 107 | if (array_key_exists($prefix,$config['namespaces'])) 108 | { 109 | $prefix = $prefix.uniqid(); 110 | } 111 | $newNsConfig[$prefix] = $n; 112 | echo "\nFound object ns $n occurs > 500 times, suggest prefix $prefix"; 113 | $config["namespaces"] = array_merge($config["namespaces"],$newNsConfig); 114 | \Tripod\Config::setConfig($config); 115 | } 116 | } 117 | } 118 | 119 | $currentSubject=$subject; // reset current subject to next subject 120 | $triples = array(); // reset triples 121 | } 122 | $triples[] = $line; 123 | } 124 | 125 | print "Suggested namespace configuration:\n\n"; 126 | 127 | /** 128 | * @param string $json 129 | * @return string 130 | */ 131 | function indent($json) { 132 | 133 | $result = ''; 134 | $pos = 0; 135 | $strLen = strlen($json); 136 | $indentStr = ' '; 137 | $newLine = "\n"; 138 | $prevChar = ''; 139 | $outOfQuotes = true; 140 | 141 | for ($i=0; $i<=$strLen; $i++) { 142 | 143 | // Grab the next character in the string. 144 | $char = substr($json, $i, 1); 145 | 146 | // Are we inside a quoted string? 147 | if ($char == '"' && $prevChar != '\\') { 148 | $outOfQuotes = !$outOfQuotes; 149 | 150 | // If this character is the end of an element, 151 | // output a new line and indent the next line. 152 | } else if(($char == '}' || $char == ']') && $outOfQuotes) { 153 | $result .= $newLine; 154 | $pos --; 155 | for ($j=0; $j<$pos; $j++) { 156 | $result .= $indentStr; 157 | } 158 | } 159 | 160 | // Add the character to the result string. 161 | $result .= $char; 162 | 163 | // If the last character was the beginning of an element, 164 | // output a new line and indent the next line. 165 | if (($char == ',' || $char == '{' || $char == '[') && $outOfQuotes) { 166 | $result .= $newLine; 167 | if ($char == '{' || $char == '[') { 168 | $pos ++; 169 | } 170 | 171 | for ($j = 0; $j < $pos; $j++) { 172 | $result .= $indentStr; 173 | } 174 | } 175 | 176 | $prevChar = $char; 177 | } 178 | 179 | return $result; 180 | } 181 | 182 | $json = json_encode(array("namespaces"=>$config["namespaces"])); 183 | 184 | print indent($json); -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | commands: 4 | setup_replica_set: 5 | parameters: 6 | set_name: 7 | type: string 8 | member_hosts: 9 | type: string 10 | steps: 11 | - run: 12 | name: Setup a replica set 13 | no_output_timeout: "2m" 14 | environment: 15 | RS_NAME: << parameters.set_name >> 16 | MEMBERS: << parameters.member_hosts >> 17 | command: | 18 | IFS=',' read -r -a MEMBERS_ARR \<<<"$MEMBERS" 19 | INITIATOR=${MEMBERS_ARR[0]} 20 | MEMBERS_JS='' 21 | for i in "${!MEMBERS_ARR[@]}"; do 22 | MEMBERS_JS="${MEMBERS_JS}$(printf '{ _id: %d, host: "%s" },' $i "${MEMBERS_ARR[$i]}")" 23 | done 24 | 25 | MONGO_CLI="$(command -v mongosh mongo | head -n1)" 26 | _mongo() { "$MONGO_CLI" --quiet --host "$@"; } 27 | 28 | for member in "${MEMBERS_ARR[@]}"; do 29 | echo "Waiting for $member" 30 | until _mongo "$member" \<<<'db.adminCommand("ping")' | grep 'ok'; do 31 | sleep 1 32 | done 33 | done 34 | 35 | echo "Initiating replica set $RS_NAME..." 36 | _mongo "$INITIATOR" \<<<"$(printf 'rs.initiate({ _id: "%s", members: [%s] })' "$RS_NAME" "$MEMBERS_JS")" | tee /dev/stderr | grep -E 'ok|already initialized' 37 | echo "Waiting for primary..." 38 | _mongo "$INITIATOR" \<<<'while (true) { if (rs.status().members.some(({ state }) => state === 1)) { break; } sleep(1000); }' 39 | echo "Waiting for secondaries..." 40 | _mongo "$INITIATOR" \<<<'while (true) { if (rs.status().members.every(({state}) => state == 1 || state == 2)) { break; } sleep(1000); }' 41 | echo "Checking status..." 42 | _mongo "$INITIATOR" \<<<'rs.status();' | tee /dev/stderr | grep "$RS_NAME" 43 | echo "Replica set configured!" 44 | 45 | check_mongodb_lib_version: 46 | steps: 47 | - run: 48 | name: Check mongodb ext+lib parity 49 | command: | 50 | php -r 'echo phpversion("mongodb"), PHP_EOL;' 51 | grep '"name": "mongodb/mongodb",' composer.lock -A1 52 | 53 | run_test: 54 | steps: 55 | - run: composer test -- --log-junit test-results/junit.xml 56 | - store_test_results: 57 | path: test-results/junit.xml 58 | - store_artifacts: 59 | path: test-results/junit.xml 60 | 61 | jobs: 62 | test: 63 | parameters: 64 | php_version: { type: string } 65 | mongo_version: { type: string } 66 | docker: 67 | - image: talis/tripod-php:<< parameters.php_version >>-latest 68 | - { name: mongodb, image: mongo:<< parameters.mongo_version >> } 69 | - { name: redis, image: redis:6.2.6 } 70 | environment: 71 | RESQUE_SERVER: redis 72 | steps: 73 | - checkout 74 | - run: composer install 75 | - check_mongodb_lib_version 76 | - run_test 77 | 78 | test-multiple-stores: 79 | docker: 80 | - image: talis/tripod-php:php74-latest 81 | - { name: mongo1, image: mongo:4.4.29 } 82 | - { name: mongo2, image: mongo:5.0.28 } 83 | - { name: redis, image: redis:6.2.6 } 84 | environment: 85 | RESQUE_SERVER: redis 86 | TRIPOD_DATASOURCE_RS1_CONFIG: | 87 | {"type":"mongo", "connection":"mongodb://mongo1:27017/", "replicaSet":""} 88 | TRIPOD_DATASOURCE_RS2_CONFIG: | 89 | {"type":"mongo", "connection":"mongodb://mongo2:27017/", "replicaSet":""} 90 | steps: 91 | - checkout 92 | - run: composer install 93 | - check_mongodb_lib_version 94 | - run_test 95 | 96 | test-replica-set-mmap: 97 | docker: 98 | - image: talis/tripod-php:php74-latest 99 | - { name: mongo1, image: mongo:3.6.23, command: mongod --storageEngine mmapv1 --smallfiles --replSet=tripod-rs } 100 | - { name: mongo2, image: mongo:3.6.23, command: mongod --storageEngine mmapv1 --smallfiles --replSet=tripod-rs } 101 | - { name: redis, image: redis:6.2.6 } 102 | environment: 103 | RESQUE_SERVER: redis 104 | TRIPOD_DATASOURCE_RS1_CONFIG: | 105 | {"type":"mongo", "connection":"mongodb://mongo1,mongo2/?retryWrites=false", "replicaSet":"tripod-rs"} 106 | TRIPOD_DATASOURCE_RS2_CONFIG: | 107 | {"type":"mongo", "connection":"mongodb://mongo1,mongo2/?retryWrites=false", "replicaSet":"tripod-rs"} 108 | steps: 109 | - checkout 110 | - setup_replica_set: 111 | set_name: tripod-rs 112 | member_hosts: mongo1,mongo2 113 | - run: composer install 114 | - check_mongodb_lib_version 115 | - run_test 116 | 117 | test-replica-set-wiredtiger: 118 | docker: 119 | - image: talis/tripod-php:php74-latest 120 | - { name: mongo1, image: mongo:5.0.28, command: mongod --replSet=tripod-rs } 121 | - { name: mongo2, image: mongo:5.0.28, command: mongod --replSet=tripod-rs } 122 | - { name: redis, image: redis:6.2.6 } 123 | environment: 124 | RESQUE_SERVER: redis 125 | TRIPOD_DATASOURCE_RS1_CONFIG: | 126 | {"type":"mongo", "connection":"mongodb://mongo1,mongo2/admin?replicaSet=tripod-rs", "replicaSet":""} 127 | TRIPOD_DATASOURCE_RS2_CONFIG: | 128 | {"type":"mongo", "connection":"mongodb://mongo1,mongo2/admin?replicaSet=tripod-rs", "replicaSet":""} 129 | steps: 130 | - checkout 131 | - setup_replica_set: 132 | set_name: tripod-rs 133 | member_hosts: mongo1,mongo2 134 | - run: composer install 135 | - check_mongodb_lib_version 136 | - run_test 137 | 138 | workflows: 139 | build_and_test: 140 | jobs: 141 | - test: 142 | name: test-php73 143 | php_version: php73 144 | mongo_version: 3.6.23 145 | - test: 146 | name: test-php74 147 | php_version: php74 148 | mongo_version: 4.0.28 149 | - test-multiple-stores 150 | - test-replica-set-mmap 151 | - test-replica-set-wiredtiger 152 | -------------------------------------------------------------------------------- /src/mongo/jobs/ApplyOperation.class.php: -------------------------------------------------------------------------------- 1 | getStat()->increment( 29 | MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 30 | count($this->args[self::SUBJECTS_KEY]) 31 | ); 32 | 33 | foreach ($this->args[self::SUBJECTS_KEY] as $subject) { 34 | $opTimer = new \Tripod\Timer(); 35 | $opTimer->start(); 36 | 37 | $impactedSubject = $this->createImpactedSubject($subject); 38 | $impactedSubject->update(); 39 | 40 | $opTimer->stop(); 41 | // stat time taken to perform operation for the given subject 42 | $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION.'.'.$subject['operation'], $opTimer->result()); 43 | 44 | /** 45 | * ApplyOperation jobs can either apply to a single resource (e.g. 'create composite for the given 46 | * resource uri) or for a specification id (i.e. regenerate all of the composites defined by the 47 | * specification). For the latter, we need to keep track of how many jobs have run so we can clean 48 | * up any stale composite documents when completed. The TRACKING_KEY value will be the JobGroup id. 49 | */ 50 | if (isset($this->args[self::TRACKING_KEY])) { 51 | $jobGroup = $this->getJobGroup($subject['storeName'], $this->args[self::TRACKING_KEY]); 52 | $jobCount = $jobGroup->incrementJobCount(-1); 53 | if ($jobCount <= 0) { 54 | // @todo Replace this with ObjectId->getTimestamp() if we upgrade Mongo driver to 1.2 55 | $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobGroup->getId(), 0, 8)) * 1000); 56 | $tripod = $this->getTripod($subject['storeName'], $subject['podName']); 57 | $count = 0; 58 | foreach ($subject['specTypes'] as $specId) { 59 | switch ($subject['operation']) { 60 | case \OP_VIEWS: 61 | $count += $tripod->getComposite(\OP_VIEWS)->deleteViewsByViewId($specId, $timestamp); 62 | break; 63 | case \OP_TABLES: 64 | $count += $tripod->getComposite(\OP_TABLES)->deleteTableRowsByTableId($specId, $timestamp); 65 | break; 66 | case \OP_SEARCH: 67 | $searchProvider = $this->getSearchProvider($tripod); 68 | $count += $searchProvider->deleteSearchDocumentsByTypeId($specId, $timestamp); 69 | break; 70 | } 71 | } 72 | $this->infoLog( 73 | '[JobGroupId ' . $jobGroup->getId()->__toString() . '] composite cleanup for ' . 74 | $subject['operation'] . ' removed ' . $count . ' stale composite documents' 75 | ); 76 | } 77 | } 78 | } 79 | } 80 | 81 | /** 82 | * Stat string for successful job timer 83 | * 84 | * @return string 85 | */ 86 | protected function getStatTimerSuccessKey() 87 | { 88 | return MONGO_QUEUE_APPLY_OPERATION_SUCCESS; 89 | } 90 | 91 | /** 92 | * Stat string for failed job increment 93 | * 94 | * @return string 95 | */ 96 | protected function getStatFailureIncrementKey() 97 | { 98 | return MONGO_QUEUE_APPLY_OPERATION_FAIL; 99 | } 100 | 101 | /** 102 | * @param \Tripod\Mongo\ImpactedSubject[] $subjects 103 | * @param string|null $queueName 104 | * @param array $otherData 105 | */ 106 | public function createJob(array $subjects, $queueName = null, $otherData = []) 107 | { 108 | $configInstance = $this->getConfigInstance(); 109 | if (!$queueName) { 110 | $queueName = $configInstance::getApplyQueueName(); 111 | } elseif (strpos($queueName, $configInstance::getApplyQueueName()) === false) { 112 | $queueName = $configInstance::getApplyQueueName() . '::' . $queueName; 113 | } 114 | 115 | $data = [ 116 | self::SUBJECTS_KEY => array_map( 117 | function (\Tripod\Mongo\ImpactedSubject $subject) { 118 | return $subject->toArray(); 119 | }, 120 | $subjects 121 | ), 122 | ]; 123 | 124 | $data = array_merge( 125 | $this->generateConfigJobArgs(), 126 | $data 127 | ); 128 | 129 | $this->submitJob($queueName, get_class($this), array_merge($otherData, $data)); 130 | } 131 | 132 | /** 133 | * For mocking 134 | * @param array $args 135 | * @return \Tripod\Mongo\ImpactedSubject 136 | */ 137 | protected function createImpactedSubject(array $args) 138 | { 139 | return new \Tripod\Mongo\ImpactedSubject( 140 | $args["resourceId"], 141 | $args["operation"], 142 | $args["storeName"], 143 | $args["podName"], 144 | $args["specTypes"] 145 | ); 146 | } 147 | 148 | /** 149 | * For mocking 150 | * 151 | * @param string $storeName Tripod store (database) name 152 | * @param string|\MongoDB\BSON\ObjectId $trackingKey JobGroup ID 153 | * @return JobGroup 154 | */ 155 | protected function getJobGroup($storeName, $trackingKey) 156 | { 157 | return new JobGroup($storeName, $trackingKey); 158 | } 159 | 160 | /** 161 | * For mocking 162 | * 163 | * @param Driver $tripod 164 | * @return \Tripod\Mongo\MongoSearchProvider 165 | */ 166 | protected function getSearchProvider(Driver $tripod) 167 | { 168 | return new \Tripod\Mongo\MongoSearchProvider($tripod); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/mongo/serializers/NQuadSerializer.class.php: -------------------------------------------------------------------------------- 1 | esc_chars = array(); 16 | $this->raw = 0; 17 | } 18 | 19 | /** 20 | * @param string|array $v 21 | * @return string 22 | */ 23 | public function getTerm($v) 24 | { 25 | if (!is_array($v)) { 26 | if (preg_match('/^\_\:/', $v)) { 27 | return $v; 28 | } 29 | if (preg_match('/^[a-z0-9]+\:[^\s\"]*$/is', $v)) { 30 | return '<' . htmlspecialchars($this->escape($v)) . '>'; 31 | } 32 | return $this->getTerm(array('type' => 'literal', 'value' => $v)); 33 | } 34 | if ($v['type'] != 'literal') { 35 | return $this->getTerm($v['value']); 36 | } 37 | /* literal */ 38 | $quot = '"'; 39 | if ($this->raw && preg_match('/\"/', $v['value'])) { 40 | $quot = "'"; 41 | if (preg_match('/\'/', $v['value'])) { 42 | $quot = '"""'; 43 | if (preg_match('/\"\"\"/', $v['value']) || preg_match('/\"$/', $v['value']) || preg_match('/^\"/', $v['value'])) { 44 | $quot = "'''"; 45 | $v['value'] = preg_replace("/'$/", "' ", $v['value']); 46 | $v['value'] = preg_replace("/^'/", " '", $v['value']); 47 | $v['value'] = str_replace("'''", '\\\'\\\'\\\'', $v['value']); 48 | } 49 | } 50 | } 51 | if ($this->raw && (strlen($quot) == 1) && preg_match('/[\x0d\x0a]/', $v['value'])) { 52 | $quot = $quot . $quot . $quot; 53 | } 54 | 55 | $suffix = isset($v['lang']) && $v['lang'] ? '@' . $v['lang'] : ''; 56 | $suffix = isset($v['datatype']) && $v['datatype'] ? '^^' . $this->getTerm($v['datatype']) : $suffix; 57 | 58 | $escaped = $this->escape($v['value']); 59 | 60 | 61 | return $quot . $escaped . $quot . $suffix; 62 | } 63 | 64 | /** 65 | * @param array $index 66 | * @param string $context 67 | * @return string 68 | */ 69 | public function getSerializedIndex($index, $context) 70 | { 71 | $r = ''; 72 | $nl = "\n"; 73 | foreach ($index as $s => $ps) { 74 | $s = $this->getTerm($s); 75 | 76 | // Fix added to ensure we do not serialize to quads 77 | // any triple where the subject is a literal 78 | if(strpos($s, "\"") === 0){ 79 | continue; 80 | } 81 | 82 | foreach ($ps as $p => $os) { 83 | $p = $this->getTerm($p); 84 | if (!is_array($os)) { /* single literal o */ 85 | $os = array(array('value' => $os, 'type' => 'literal')); 86 | } 87 | foreach ($os as $o) { 88 | $o = $this->getTerm($o); 89 | $r .= $r ? $nl : ''; 90 | $r .= $s . ' ' . $p . ' ' . $o; 91 | if($context != null) { 92 | $r .= ' <'.$context.'>'; 93 | } 94 | $r .= ' .'; 95 | } 96 | } 97 | } 98 | return $r . $nl; 99 | } 100 | 101 | /** 102 | * @param string $v 103 | * @return string 104 | */ 105 | function escape($v) 106 | { 107 | $r = ''; 108 | $v = (strpos(utf8_decode(str_replace('?', '', $v)), '?') === false) ? utf8_decode($v) : $v; 109 | if ($this->raw) { 110 | return $v; 111 | } 112 | 113 | for ($i = 0, $i_max = strlen($v); $i < $i_max; $i++) { 114 | $c = $v[$i]; 115 | if (!isset($this->esc_chars[$c])) { 116 | $this->esc_chars[$c] = $this->getEscapedChar($c, $this->getCharNo($c)); 117 | } 118 | $r .= $this->esc_chars[$c]; 119 | } 120 | return $r; 121 | } 122 | 123 | 124 | /** 125 | * @param string $c 126 | * @return int 127 | */ 128 | function getCharNo($c) 129 | { 130 | $c_utf = utf8_encode($c); 131 | $bl = strlen($c_utf); /* binary length */ 132 | $r = 0; 133 | switch ($bl) { 134 | case 1: /* 0####### (0-127) */ 135 | $r = ord($c_utf); 136 | break; 137 | case 2: /* 110##### 10###### = 192+x 128+x */ 138 | $r = ((ord($c_utf[0]) - 192) * 64) + (ord($c_utf[1]) - 128); 139 | break; 140 | case 3: /* 1110#### 10###### 10###### = 224+x 128+x 128+x */ 141 | $r = ((ord($c_utf[0]) - 224) * 4096) + ((ord($c_utf[1]) - 128) * 64) + (ord($c_utf[2]) - 128); 142 | break; 143 | case 4: /* 1111#### 10###### 10###### 10###### = 240+x 128+x 128+x 128+x */ 144 | $r = ((ord($c_utf[0]) - 240) * 262144) + ((ord($c_utf[1]) - 128) * 4096) + ((ord($c_utf[2]) - 128) * 64) + (ord($c_utf[3]) - 128); 145 | break; 146 | } 147 | return $r; 148 | } 149 | 150 | /** 151 | * @param string $c 152 | * @param int $no 153 | * @return string 154 | */ 155 | function getEscapedChar($c, $no) 156 | { /*see http://www.w3.org/TR/rdf-testcases/#ntrip_strings */ 157 | if ($no < 9) return "\\u" . sprintf('%04X', $no); /* #x0-#x8 (0-8) */ 158 | if ($no == 9) return '\t'; /* #x9 (9) */ 159 | if ($no == 10) return '\n'; /* #xA (10) */ 160 | if ($no < 13) return "\\u" . sprintf('%04X', $no); /* #xB-#xC (11-12) */ 161 | if ($no == 13) return '\r'; /* #xD (13) */ 162 | if ($no < 32) return "\\u" . sprintf('%04X', $no); /* #xE-#x1F (14-31) */ 163 | if ($no < 34) return $c; /* #x20-#x21 (32-33) */ 164 | if ($no == 34) return '\"'; /* #x22 (34) */ 165 | if ($no < 92) return $c; /* #x23-#x5B (35-91) */ 166 | if ($no == 92) return '\\\\'; /* #x5C (92) */ 167 | if ($no < 127) return $c; /* #x5D-#x7E (93-126) */ 168 | if ($no < 65536) return "\\u" . sprintf('%04X', $no); /* #x7F-#xFFFF (128-65535) */ 169 | if ($no < 1114112) return "\\U" . sprintf('%08X', $no); /* #x10000-#x10FFFF (65536-1114111) */ 170 | return ''; /* not defined => ignore */ 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/classes/StatsD.class.php: -------------------------------------------------------------------------------- 1 | host = $host; 28 | $this->port = $port; 29 | $this->setPrefix($prefix); 30 | } 31 | 32 | /** 33 | * @param string $operation 34 | * @param int $inc 35 | * @return void 36 | */ 37 | public function increment($operation, $inc=1) 38 | { 39 | $this->send( 40 | $this->generateStatData($operation, $inc."|c") 41 | ); 42 | } 43 | 44 | /** 45 | * @param string $operation 46 | * @param number $duration 47 | * @return mixed 48 | */ 49 | public function timer($operation, $duration) 50 | { 51 | $this->send( 52 | $this->generateStatData($operation, array("1|c","$duration|ms")) 53 | ); 54 | } 55 | 56 | /** 57 | * Record an arbitrary value 58 | * 59 | * @param string $operation 60 | * @param mixed $value 61 | */ 62 | public function gauge($operation, $value) 63 | { 64 | $this->send( 65 | $this->generateStatData($operation, $value.'|g') 66 | ); 67 | } 68 | 69 | /** 70 | * @return array 71 | */ 72 | public function getConfig() 73 | { 74 | return array( 75 | 'class'=>get_class($this), 76 | 'config'=>array( 77 | 'host'=>$this->host, 78 | 'port'=>$this->port, 79 | 'prefix'=>$this->prefix 80 | ) 81 | ); 82 | } 83 | 84 | /** 85 | * Sends the stat(s) using UDP protocol 86 | * @param $data 87 | * @param int $sampleRate 88 | */ 89 | protected function send($data, $sampleRate=1) { 90 | $sampledData = array(); 91 | if ($sampleRate < 1) { 92 | foreach ($data as $stat => $value) { 93 | if ((mt_rand() / mt_getrandmax()) <= $sampleRate) { 94 | $sampledData[$stat] = "$value|@$sampleRate"; 95 | } 96 | } 97 | } else { 98 | $sampledData = $data; 99 | } 100 | if (empty($sampledData)) { return; } 101 | try 102 | { 103 | if (!empty($this->host)) // if host is configured, send.. 104 | { 105 | $fp = fsockopen("udp://{$this->host}", $this->port); 106 | if (! $fp) { return; } 107 | // make this a non blocking stream 108 | stream_set_blocking($fp, false); 109 | foreach ($sampledData as $stat => $value) 110 | { 111 | if (is_array($value)) 112 | { 113 | foreach ($value as $v) 114 | { 115 | fwrite($fp, "$stat:$v"); 116 | } 117 | } 118 | else 119 | { 120 | fwrite($fp, "$stat:$value"); 121 | } 122 | } 123 | fclose($fp); 124 | } 125 | } 126 | catch (\Exception $e) 127 | { 128 | } 129 | } 130 | 131 | /** 132 | * @param array $config 133 | * @return StatsD 134 | */ 135 | public static function createFromConfig(array $config) 136 | { 137 | if(isset($config['config'])) 138 | { 139 | $config = $config['config']; 140 | } 141 | 142 | $host = (isset($config['host']) ? $config['host'] : null); 143 | $port = (isset($config['port']) ? $config['port'] : null); 144 | $prefix = (isset($config['prefix']) ? $config['prefix'] : ''); 145 | return new self($host, $port, $prefix); 146 | } 147 | 148 | /** 149 | * @return string 150 | */ 151 | public function getPrefix() 152 | { 153 | return $this->prefix; 154 | } 155 | 156 | /** 157 | * @param string $prefix 158 | * @throws \InvalidArgumentException 159 | */ 160 | public function setPrefix($prefix) 161 | { 162 | if($this->isValidPathValue($prefix)) 163 | { 164 | $this->prefix = $prefix; 165 | } 166 | else 167 | { 168 | throw new \InvalidArgumentException('Invalid prefix supplied'); 169 | } 170 | } 171 | 172 | /** 173 | * @return int|string 174 | */ 175 | public function getPort() 176 | { 177 | return $this->port; 178 | } 179 | 180 | /** 181 | * @param int|string $port 182 | */ 183 | public function setPort($port) 184 | { 185 | $this->port = $port; 186 | } 187 | 188 | /** 189 | * @return string 190 | */ 191 | public function getHost() 192 | { 193 | return $this->host; 194 | } 195 | 196 | /** 197 | * @param string $host 198 | */ 199 | public function setHost($host) 200 | { 201 | $this->host = $host; 202 | } 203 | 204 | /** 205 | * This method combines the by database and aggregate stats to send to StatsD. The return will look something list: 206 | * { 207 | * "{prefix}.tripod.group_by_db.{storeName}.{stat}"=>"1|c", 208 | * "{prefix}.tripod.{stat}"=>"1|c" 209 | * } 210 | * 211 | * @param string $operation 212 | * @param string|array $value 213 | * @return array An associative array of the grouped_by_database and aggregate stats 214 | */ 215 | protected function generateStatData($operation, $value) 216 | { 217 | $data = array(); 218 | foreach($this->getStatsPaths() as $path) 219 | { 220 | $data[$path . ".$operation"]=$value; 221 | } 222 | return $data; 223 | } 224 | 225 | /** 226 | * @return string 227 | */ 228 | public function getPivotValue() 229 | { 230 | return $this->pivotValue; 231 | } 232 | 233 | /** 234 | * @param string $pivotValue 235 | * @throws \InvalidArgumentException 236 | */ 237 | public function setPivotValue($pivotValue) 238 | { 239 | if($this->isValidPathValue($pivotValue)) 240 | { 241 | $this->pivotValue = $pivotValue; 242 | } 243 | else 244 | { 245 | throw new \InvalidArgumentException('Invalid pivot value supplied'); 246 | } 247 | } 248 | 249 | /** 250 | * @return array 251 | */ 252 | protected function getStatsPaths() 253 | { 254 | return array_values(array_filter([$this->getAggregateStatPath()])); 255 | } 256 | 257 | /** 258 | * @return string 259 | */ 260 | protected function getAggregateStatPath() 261 | { 262 | return (empty($this->prefix) ? STAT_CLASS : $this->prefix . '.' . STAT_CLASS); 263 | } 264 | 265 | /** 266 | * StatsD paths cannot start with, end with, or have more than one consecutive '.' 267 | * @param $value 268 | * @return bool 269 | */ 270 | protected function isValidPathValue($value) 271 | { 272 | return (preg_match("/(^\.)|(\.\.+)|(\.$)/", $value) === 0); 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /test/unit/mongo/ConfigGeneratorTest.php: -------------------------------------------------------------------------------- 1 | config = [ 15 | 'class' => 'TestConfigGenerator', 16 | 'filename' => dirname(__FILE__) . '/data/config.json', 17 | ]; 18 | Tripod\Config::setConfig($this->config); 19 | } 20 | 21 | public function testCreateFromConfig() 22 | { 23 | /** @var TestConfigGenerator $instance */ 24 | $instance = Tripod\Config::getInstance(); 25 | $this->assertInstanceOf(TestConfigGenerator::class, $instance); 26 | $this->assertInstanceOf(Tripod\Mongo\Config::class, $instance); 27 | $this->assertInstanceOf(Tripod\ITripodConfigSerializer::class, $instance); 28 | $this->assertEquals( 29 | ['CBD_testing', 'CBD_test_related_content', 'CBD_testing_2'], 30 | $instance->getPods('tripod_php_testing') 31 | ); 32 | } 33 | 34 | public function testSerializeConfig() 35 | { 36 | /** @var TestConfigGenerator $instance */ 37 | $instance = Tripod\Config::getInstance(); 38 | $this->assertEquals($this->config, $instance->serialize()); 39 | } 40 | 41 | public function testConfigGeneratorsSerializedInDiscoverJobs() 42 | { 43 | $originalGraph = new Tripod\ExtendedGraph(); 44 | $originalGraph->add_resource_triple('http://example.com/1', RDF_TYPE, RDFS_CLASS); 45 | 46 | $newGraph = new Tripod\ExtendedGraph(); 47 | $newGraph->add_resource_triple('http://example.com/1', RDF_TYPE, OWL_CLASS); 48 | $subjectsAndPredicatesOfChange = ['http://example.com/1' => [RDF_TYPE]]; 49 | 50 | $tripod = $this->getMockBuilder(Tripod\Mongo\Driver::class) 51 | ->onlyMethods(['getDataUpdater']) 52 | ->setConstructorArgs( 53 | ['CBD_testing', 'tripod_php_testing'] 54 | ) 55 | ->getMock(); 56 | 57 | $updates = $this->getMockBuilder(Tripod\Mongo\Updates::class) 58 | ->onlyMethods( 59 | [ 60 | 'applyHooks', 61 | 'storeChanges', 62 | 'setReadPreferenceToPrimary', 63 | 'processSyncOperations', 64 | 'getDiscoverImpactedSubjects', 65 | 'resetOriginalReadPreference', 66 | ] 67 | ) 68 | ->setConstructorArgs([$tripod]) 69 | ->getMock(); 70 | 71 | $discoverJob = $this->getMockBuilder(DiscoverImpactedSubjects::class) 72 | ->onlyMethods(['createJob']) 73 | ->getMock(); 74 | 75 | $tripod->expects($this->once())->method('getDataUpdater')->will($this->returnValue($updates)); 76 | $updates->expects($this->once())->method('getDiscoverImpactedSubjects')->will($this->returnValue($discoverJob)); 77 | 78 | $updates->expects($this->once())->method('storeChanges')->will( 79 | $this->returnValue( 80 | ['transaction_id' => uniqid(), 'subjectsAndPredicatesOfChange' => $subjectsAndPredicatesOfChange] 81 | ) 82 | ); 83 | 84 | $discoverJob->expects($this->once())->method('createJob') 85 | ->with([ 86 | 'changes' => $subjectsAndPredicatesOfChange, 87 | 'operations' => [OP_TABLES, OP_SEARCH], 88 | 'storeName' => 'tripod_php_testing', 89 | 'podName' => 'CBD_testing', 90 | 'contextAlias' => 'http://talisaspire.com/', 91 | 'statsConfig' => [], 92 | ]); 93 | 94 | $tripod->saveChanges( 95 | $originalGraph, 96 | $newGraph 97 | ); 98 | } 99 | 100 | public function testSerializedConfigGeneratorsSentToApplyJobs() 101 | { 102 | $subjectsAndPredicatesOfChange = ['http://example.com/1' => [RDF_TYPE]]; 103 | $impactedSubjects = [ 104 | new ImpactedSubject( 105 | [_ID_RESOURCE => 'http://example.com/1', _ID_CONTEXT => 'http://talisaspire.com/'], 106 | OP_VIEWS, 107 | 'tripod_php_testing', 108 | 'CBD_testing', 109 | ['v_resource_full'] 110 | ), 111 | ]; 112 | $jobArgs = [ 113 | DiscoverImpactedSubjects::STORE_NAME_KEY => 'tripod_php_testing', 114 | DiscoverImpactedSubjects::POD_NAME_KEY => 'CBD_testing', 115 | DiscoverImpactedSubjects::CHANGES_KEY => $subjectsAndPredicatesOfChange, 116 | DiscoverImpactedSubjects::OPERATIONS_KEY => [OP_VIEWS], 117 | DiscoverImpactedSubjects::CONTEXT_ALIAS_KEY => 'http://talisaspire.com/', 118 | JobBase::TRIPOD_CONFIG_GENERATOR => $this->config, 119 | ]; 120 | 121 | $tripod = $this->getMockBuilder(Tripod\Mongo\Driver::class) 122 | ->onlyMethods(['getComposite']) 123 | ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) 124 | ->getMock(); 125 | 126 | $views = $this->getMockBuilder(Tripod\Mongo\Composites\Views::class) 127 | ->onlyMethods(['getImpactedSubjects']) 128 | ->disableOriginalConstructor() 129 | ->getMock(); 130 | 131 | $tripod->expects($this->once())->method('getComposite') 132 | ->with(OP_VIEWS) 133 | ->will($this->returnValue($views)); 134 | 135 | $views->expects($this->once())->method('getImpactedSubjects')->will($this->returnValue($impactedSubjects)); 136 | 137 | $discoverJob = $this->getMockBuilder(DiscoverImpactedSubjects::class) 138 | ->onlyMethods(['getTripod', 'getApplyOperation']) 139 | ->getMock(); 140 | 141 | $applyJob = $this->getMockBuilder(ApplyOperation::class) 142 | ->onlyMethods(['submitJob']) 143 | ->setMockClassName('ApplyOperation_TestConfigGenerator') 144 | ->getMock(); 145 | $discoverJob->args = $jobArgs; 146 | $discoverJob->job = (object) ['payload' => ['id' => uniqid()]]; 147 | $discoverJob->expects($this->once())->method('getTripod')->will($this->returnValue($tripod)); 148 | $discoverJob->expects($this->once())->method('getApplyOperation')->will($this->returnValue($applyJob)); 149 | $configInstance = Tripod\Config::getInstance(); 150 | $applyJob->expects($this->once())->method('submitJob') 151 | ->with( 152 | $configInstance::getApplyQueueName(), 153 | 'ApplyOperation_TestConfigGenerator', 154 | [ 155 | ApplyOperation::SUBJECTS_KEY => [ 156 | [ 157 | 'resourceId' => [ 158 | _ID_RESOURCE => 'http://example.com/1', 159 | _ID_CONTEXT => 'http://talisaspire.com/', 160 | ], 161 | 'operation' => OP_VIEWS, 162 | 'specTypes' => ['v_resource_full'], 163 | 'storeName' => 'tripod_php_testing', 164 | 'podName' => 'CBD_testing', 165 | ], 166 | ], 167 | JobBase::TRIPOD_CONFIG_GENERATOR => $this->config, 168 | ] 169 | ); 170 | $discoverJob->setUp(); 171 | $discoverJob->perform(); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/IDriver.php: -------------------------------------------------------------------------------- 1 | $newGraph 117 | * @param ExtendedGraph $oldGraph 118 | * @param ExtendedGraph $newGraph 119 | * @param null|string $context 120 | * @param null|string $description 121 | * @return bool true or throws exception on error 122 | */ 123 | public function saveChanges(ExtendedGraph $oldGraph, ExtendedGraph $newGraph,$context=null,$description=null); 124 | 125 | /** 126 | * Register an event hook, which will be executed when the event fires. 127 | * @param $eventType 128 | * @param IEventHook $hook 129 | */ 130 | public function registerHook($eventType,IEventHook $hook); 131 | 132 | /* START Deprecated methods that will be removed in 1.x.x */ 133 | 134 | /** 135 | * Return (DESCRIBE) according to a filter 136 | * @deprecated Use graph() instead 137 | * @param array $filter conditions to filter by 138 | * @return ExtendedGraph 139 | */ 140 | public function describe($filter); 141 | 142 | /** 143 | * Generates table rows 144 | * @deprecated calling save will generate table rows - this method seems to be only used in tests and does not belong on the interface 145 | * @param $tableType 146 | * @param null|string $resource 147 | * @param null|string $context 148 | */ 149 | public function generateTableRows($tableType,$resource=null,$context=null); 150 | 151 | /** 152 | * Submits search params to configured search provider 153 | * the params array must contain the following keys 154 | * -q the query string to search for 155 | * -type the search document type to restrict results to, in other words _id.type 156 | * -indices an array of indices (from spec) to match query terms against, must specify at least one 157 | * -fields an array of the fields (from spec) you want included in the search results, must specify at least one 158 | * -limit integer the number of results to return per page 159 | * -offset the offset to skip to when returning results 160 | * 161 | * this method looks for the above keys in the params array and naively passes them to the search provider which will 162 | * throw SearchException if any of the params are invalid 163 | * 164 | * @deprecated Search will be removed from a future version of Tripod as its functionality is equivalent to tables 165 | * @param Array $params 166 | * @throws \Tripod\Exceptions\Exception - if search provider cannot be found 167 | * @throws \Tripod\Exceptions\SearchException - if something goes wrong 168 | * @return Array results 169 | */ 170 | public function search(Array $params); 171 | 172 | /** 173 | * Get any documents that were left in a locked state 174 | * @deprecated this is a feature of the mongo implementation - this method will move from the interface to the mongo-specific Driver class soon. 175 | * @param null|string $fromDateTime strtotime compatible string 176 | * @param null|string $tillDateTime strtotime compatible string 177 | * @return array of locked documents 178 | */ 179 | public function getLockedDocuments($fromDateTime =null , $tillDateTime = null); 180 | 181 | /** 182 | * Remove any inert locks left by a given transaction 183 | * @deprecated this is a feature of the mongo implementation - this method will move from the interface to the mongo-specific Driver class soon. 184 | * @param string $transaction_id 185 | * @param string $reason 186 | * @return bool true or throws exception on error 187 | */ 188 | public function removeInertLocks($transaction_id, $reason); 189 | 190 | /* END Deprecated methods that will be removed in 1.x.x */ 191 | 192 | } 193 | -------------------------------------------------------------------------------- /src/mongo/util/IndexUtils.class.php: -------------------------------------------------------------------------------- 1 | getConfig(); 20 | $dbs = ($storeName==null) ? $config->getDbs() : array($storeName); 21 | $reindexedCollections = []; 22 | foreach ($dbs as $storeName) 23 | { 24 | $collections = $config->getIndexesGroupedByCollection($storeName); 25 | foreach ($collections as $collectionName=>$indexes) 26 | { 27 | // Don't do this for composites, which could be anywhere 28 | if(in_array($collectionName, array(TABLE_ROWS_COLLECTION,VIEWS_COLLECTION,SEARCH_INDEX_COLLECTION))) 29 | { 30 | continue; 31 | } 32 | 33 | if ($reindex) 34 | { 35 | $collection = $config->getCollectionForCBD($storeName, $collectionName); 36 | if (!in_array($collection->getNamespace(), $reindexedCollections)) { 37 | $collection->dropIndexes(); 38 | $reindexedCollections[] = $collection->getNamespace(); 39 | } 40 | } 41 | 42 | foreach ($indexes as $indexName=>$fields) 43 | { 44 | $indexName = substr($indexName,0,127); // ensure max 128 chars 45 | 46 | $indexOptions = [ 47 | 'background'=>$background 48 | ]; 49 | 50 | if (!is_numeric($indexName)) 51 | { 52 | // Named index vs. unnamed index 53 | $indexOptions['name'] = $indexName; 54 | } 55 | 56 | $indexKeys = array_keys($fields); 57 | if (is_numeric($indexKeys[0])) { 58 | // New format config - two arrays, where second is index options (e.g. unique=>true, sparse=>true) 59 | $indexFields = $fields[0]; 60 | $indexOptions = array_merge($indexOptions, $fields[1]); 61 | } else { 62 | // Standard format config - single array 63 | $indexFields = $fields; 64 | } 65 | 66 | $config->getCollectionForCBD($storeName, $collectionName) 67 | ->createIndex( 68 | $indexFields, 69 | $indexOptions 70 | ); 71 | } 72 | } 73 | 74 | // Index views 75 | foreach($config->getViewSpecifications($storeName) as $viewId=>$spec) 76 | { 77 | $collection = $config->getCollectionForView($storeName, $viewId); 78 | if($collection) 79 | { 80 | $indexes = [ 81 | [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1], 82 | [_ID_KEY.'.'._ID_TYPE => 1], 83 | ['value.'._IMPACT_INDEX => 1], 84 | [\_CREATED_TS => 1] 85 | ]; 86 | if(isset($spec['ensureIndexes'])) 87 | { 88 | $indexes = array_merge($indexes, $spec['ensureIndexes']); 89 | } 90 | if ($reindex) 91 | { 92 | if (!in_array($collection->getNamespace(), $reindexedCollections)) { 93 | $collection->dropIndexes(); 94 | $reindexedCollections[] = $collection->getNamespace(); 95 | } 96 | } 97 | foreach($indexes as $index) 98 | { 99 | $collection->createIndex( 100 | $index, 101 | array( 102 | "background"=>$background 103 | ) 104 | ); 105 | } 106 | } 107 | } 108 | 109 | // Index table rows 110 | foreach($config->getTableSpecifications($storeName) as $tableId=>$spec) 111 | { 112 | $collection = $config->getCollectionForTable($storeName, $tableId); 113 | if($collection) 114 | { 115 | $indexes = [ 116 | [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1], 117 | [_ID_KEY.'.'._ID_TYPE => 1], 118 | ['value.'._IMPACT_INDEX => 1], 119 | [\_CREATED_TS => 1] 120 | ]; 121 | if(isset($spec['ensureIndexes'])) 122 | { 123 | $indexes = array_merge($indexes, $spec['ensureIndexes']); 124 | } 125 | if ($reindex) 126 | { 127 | if (!in_array($collection->getNamespace(), $reindexedCollections)) { 128 | $collection->dropIndexes(); 129 | $reindexedCollections[] = $collection->getNamespace(); 130 | } 131 | } 132 | foreach($indexes as $index) 133 | { 134 | $collection->createIndex( 135 | $index, 136 | array( 137 | "background"=>$background 138 | ) 139 | ); 140 | } 141 | } 142 | } 143 | 144 | // index search documents 145 | foreach($config->getSearchDocumentSpecifications($storeName) as $searchId=>$spec) 146 | { 147 | $collection = $config->getCollectionForSearchDocument($storeName, $searchId); 148 | if($collection) 149 | { 150 | $indexes = [ 151 | [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1], 152 | [_ID_KEY.'.'._ID_TYPE => 1], 153 | [_IMPACT_INDEX => 1], 154 | [\_CREATED_TS => 1] 155 | ]; 156 | 157 | if($reindex) 158 | { 159 | if (!in_array($collection->getNamespace(), $reindexedCollections)) { 160 | $collection->dropIndexes(); 161 | $reindexedCollections[] = $collection->getNamespace(); 162 | } 163 | } 164 | foreach($indexes as $index) 165 | { 166 | $collection->createIndex( 167 | $index, 168 | array( 169 | "background"=>$background 170 | ) 171 | ); 172 | } 173 | } 174 | } 175 | } 176 | } 177 | 178 | /** 179 | * returns mongo tripod config instance, this method aids helps with 180 | * testing. 181 | * @return \Tripod\Mongo\Config 182 | */ 183 | protected function getConfig() 184 | { 185 | return \Tripod\Config::getInstance(); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /test/unit/mongo/MongoTripodDocumentStructureTest.php: -------------------------------------------------------------------------------- 1 | tripodTransactionLog = new Tripod\Mongo\TransactionLog(); 23 | $this->tripodTransactionLog->purgeAllTransactions(); 24 | 25 | $this->tripod = $this->getMockBuilder(Tripod\Mongo\Driver::class) 26 | ->onlyMethods([]) 27 | ->setConstructorArgs(['CBD_testing', 'tripod_php_testing', ['defaultContext' => 'http://talisaspire.com/']]) 28 | ->getMock(); 29 | 30 | $this->getTripodCollection($this->tripod)->drop(); 31 | $this->tripod->setTransactionLog($this->tripodTransactionLog); 32 | 33 | $this->loadResourceDataViaTripod(); 34 | } 35 | 36 | public function testDocumentContainsDefaultProperties() 37 | { 38 | $id = ['r' => 'http://talisaspire.com/resources/testDocument', 'c' => 'http://talisaspire.com/']; 39 | 40 | $graph = new Tripod\Mongo\MongoGraph(); 41 | $graph->add_literal_triple($id['r'], $graph->qname_to_uri('searchterms:title'), 'TEST TITLE'); 42 | $this->tripod->saveChanges(new Tripod\Mongo\MongoGraph(), $graph); 43 | 44 | $this->assertDocumentExists($id); 45 | $this->assertDocumentHasProperty($id, _VERSION, 0); 46 | $this->assertDocumentHasProperty($id, _UPDATED_TS); 47 | $this->assertDocumentHasProperty($id, _CREATED_TS); 48 | } 49 | 50 | public function testDocumentTimeStampsAreUpdatedCorrectlyAfterMultipleWritesAndDelete() 51 | { 52 | // create an initial document 53 | $id = ['r' => 'http://talisaspire.com/resources/testDocument', 'c' => 'http://talisaspire.com/']; 54 | 55 | $graph = new Tripod\Mongo\MongoGraph(); 56 | $graph->add_literal_triple($id['r'], $graph->qname_to_uri('searchterms:title'), 'TEST TITLE'); 57 | $this->tripod->saveChanges(new Tripod\Mongo\MongoGraph(), $graph); 58 | // assert that it is at version 0 59 | $this->assertDocumentExists($id); 60 | $this->assertDocumentHasProperty($id, _VERSION, 0); 61 | 62 | // retrieve the document from mongo ( rather than the graph ) capture the updated and created ts 63 | $document = $this->getDocument($id); 64 | $_updated_ts = $document[_UPDATED_TS]; 65 | $_created_ts = $document[_CREATED_TS]; 66 | 67 | sleep(1); // have to sleep to make sure ->sec will be greater between writes. 68 | 69 | // change document through tripod 70 | $newGraph = new Tripod\Mongo\MongoGraph(); 71 | $newGraph->add_literal_triple($id['r'], $graph->qname_to_uri('searchterms:title'), 'CHANGED TITLE'); 72 | $this->tripod->saveChanges($graph, $newGraph); 73 | 74 | // assert that it is at version 1 75 | $this->assertDocumentExists($id); 76 | $this->assertDocumentHasProperty($id, _VERSION, 1); 77 | 78 | // assert that the $_updated_ts has changed, but the created_ts is the same 79 | $updated_document = $this->getDocument($id); 80 | $this->assertEquals($_created_ts, $updated_document[_CREATED_TS]); 81 | $this->assertNotEquals($_updated_ts->__toString(), $updated_document[_UPDATED_TS]->__toString()); 82 | // assert that the seconds for the updated document _updated_ts is greated than the first version 83 | 84 | $this->assertGreaterThan($_updated_ts->__toString(), $updated_document[_UPDATED_TS]->__toString()); 85 | 86 | sleep(1); 87 | 88 | // update again 89 | $finalGraph = new Tripod\Mongo\MongoGraph(); 90 | $finalGraph->add_literal_triple($id['r'], $graph->qname_to_uri('searchterms:title'), 'CHANGED TITLE AGAIN'); 91 | $this->tripod->saveChanges($newGraph, $finalGraph); 92 | 93 | // assert that it is at version 2 94 | $this->assertDocumentExists($id); 95 | $this->assertDocumentHasProperty($id, _VERSION, 2); 96 | 97 | // assert that the $_updated_ts has changed, but the created_ts is the same 98 | $final_document = $this->getDocument($id); 99 | $this->assertEquals($updated_document[_CREATED_TS], $final_document[_CREATED_TS]); 100 | $this->assertNotEquals($updated_document[_UPDATED_TS]->__toString(), $final_document[_UPDATED_TS]->__toString()); 101 | $this->assertGreaterThan($updated_document[_UPDATED_TS]->__toString(), $final_document[_UPDATED_TS]->__toString()); 102 | 103 | sleep(1); 104 | 105 | // now delete through tripod, only the _ID, _VERSION, _UPDATED_TS and _CREATED_TS properties should exist on the document 106 | // updated ts will have changed the created should not have 107 | $this->tripod->saveChanges($finalGraph, new Tripod\Mongo\MongoGraph()); 108 | 109 | $this->assertDocumentExists($id); 110 | $deleted_document = $this->getDocument($id); 111 | $this->assertDocumentHasProperty($id, _VERSION); 112 | $this->assertDocumentHasProperty($id, _UPDATED_TS); 113 | $this->assertDocumentHasProperty($id, _CREATED_TS); 114 | $this->assertDocumentDoesNotHaveProperty($id, 'searchterms:title'); 115 | 116 | $this->assertEquals($final_document[_CREATED_TS], $deleted_document[_CREATED_TS]); 117 | $this->assertNotEquals($final_document[_UPDATED_TS]->__toString(), $deleted_document[_UPDATED_TS]->__toString()); 118 | $this->assertGreaterThan($final_document[_UPDATED_TS]->__toString(), $deleted_document[_UPDATED_TS]->__toString()); 119 | } 120 | 121 | /** 122 | * This test verifies that if a document was previously added to mongo without any timestamps i.e. _UPDATED_TS and _CREATED_TS 123 | * then on a tripod write only the _UPDATED_TS will be added to the document 124 | */ 125 | public function testOnlyDocumentUpdatedTimestampIsAddedToDocumentThatDidntHaveTimestampsToBeginWith() 126 | { 127 | // add the initial document, but not through Driver! 128 | $_id = ['r' => 'http://talisaspire.com/resources/testDocument2', 'c' => 'http://talisaspire.com/']; 129 | $document = [ 130 | '_id' => $_id, 131 | 'dct:title' => ['l' => 'some title'], 132 | '_version' => 0, 133 | ]; 134 | 135 | // verify initial document before we proceed, should have the triple we added, and a _version but not a 136 | // _UPDATED_TS or a _CREATED_TS 137 | $this->addDocument($document); 138 | $this->assertDocumentExists($_id); 139 | $this->assertDocumentHasProperty($_id, _VERSION, 0); 140 | $this->assertDocumentHasProperty($_id, 'dct:title', ['l' => 'some title']); 141 | $this->assertDocumentDoesNotHaveProperty($_id, _UPDATED_TS); 142 | $this->assertDocumentDoesNotHaveProperty($_id, _CREATED_TS); 143 | 144 | // change the document through tripod, for this im just doing a new addition 145 | $graph = new Tripod\Mongo\MongoGraph(); 146 | $graph->add_literal_triple($_id['r'], $graph->qname_to_uri('searchterms:title'), 'a new property'); 147 | $this->tripod->saveChanges(new Tripod\Mongo\MongoGraph(), $graph); 148 | 149 | // Now assert, document should contain the additiona triple we added, an updated _version. 150 | // Should now also contain an _UPDATED_TS but not a _CREATED_TS 151 | $this->assertDocumentExists($_id); 152 | $this->assertDocumentHasProperty($_id, _VERSION, 1); 153 | $this->assertDocumentHasProperty($_id, _UPDATED_TS); 154 | $this->assertDocumentHasProperty($_id, 'dct:title', ['l' => 'some title']); 155 | $this->assertDocumentHasProperty($_id, 'searchterms:title', ['l' => 'a new property']); 156 | $this->assertDocumentDoesNotHaveProperty($_id, _CREATED_TS); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/mongo/jobs/DiscoverImpactedSubjects.class.php: -------------------------------------------------------------------------------- 1 | getTripod( 49 | $this->args[self::STORE_NAME_KEY], 50 | $this->args[self::POD_NAME_KEY], 51 | $this->getTripodOptions() 52 | ); 53 | 54 | $operations = $this->args[self::OPERATIONS_KEY]; 55 | 56 | $subjectsAndPredicatesOfChange = $this->args[self::CHANGES_KEY]; 57 | 58 | $this->subjectCount = 0; 59 | foreach ($operations as $op) { 60 | /** @var \Tripod\Mongo\Composites\IComposite $composite */ 61 | $composite = $tripod->getComposite($op); 62 | $modifiedSubjects = $composite->getImpactedSubjects( 63 | $subjectsAndPredicatesOfChange, 64 | $this->args[self::CONTEXT_ALIAS_KEY] 65 | ); 66 | if (!empty($modifiedSubjects)) { 67 | $configInstance = $this->getConfigInstance(); 68 | /* @var $subject \Tripod\Mongo\ImpactedSubject */ 69 | foreach ($modifiedSubjects as $subject) { 70 | $this->subjectCount++; 71 | $subjectTimer = new \Tripod\Timer(); 72 | $subjectTimer->start(); 73 | if (isset($this->args[self::QUEUE_KEY]) || count($subject->getSpecTypes()) == 0) { 74 | if (isset($this->args[self::QUEUE_KEY])) { 75 | $queueName = $this->args[self::QUEUE_KEY]; 76 | } else { 77 | $queueName = $configInstance::getApplyQueueName(); 78 | } 79 | $this->addSubjectToQueue($subject, $queueName); 80 | } else { 81 | $specsGroupedByQueue = array(); 82 | foreach ($subject->getSpecTypes() as $specType) { 83 | $spec = null; 84 | switch ($subject->getOperation()) { 85 | case OP_VIEWS: 86 | $spec = $configInstance->getViewSpecification( 87 | $this->args[self::STORE_NAME_KEY], 88 | $specType 89 | ); 90 | break; 91 | case OP_TABLES: 92 | $spec = $configInstance->getTableSpecification( 93 | $this->args[self::STORE_NAME_KEY], 94 | $specType 95 | ); 96 | break; 97 | case OP_SEARCH: 98 | $spec = $configInstance->getSearchDocumentSpecification( 99 | $this->args[self::STORE_NAME_KEY], 100 | $specType 101 | ); 102 | break; 103 | } 104 | if (!$spec || !isset($spec['queue'])) { 105 | if (!$spec) { 106 | $spec = array(); 107 | } 108 | $spec['queue'] = $configInstance::getApplyQueueName(); 109 | } 110 | if (!isset($specsGroupedByQueue[$spec['queue']])) { 111 | $specsGroupedByQueue[$spec['queue']] = array(); 112 | } 113 | $specsGroupedByQueue[$spec['queue']][] = $specType; 114 | } 115 | 116 | foreach ($specsGroupedByQueue as $queueName => $specs) { 117 | $queuedSubject = new \Tripod\Mongo\ImpactedSubject( 118 | $subject->getResourceId(), 119 | $subject->getOperation(), 120 | $subject->getStoreName(), 121 | $subject->getPodName(), 122 | $specs 123 | ); 124 | 125 | $this->addSubjectToQueue($queuedSubject, $queueName); 126 | } 127 | } 128 | $subjectTimer->stop(); 129 | // stat time taken to discover impacted subjects for the given subject of change 130 | $this->getStat()->timer(MONGO_QUEUE_DISCOVER_SUBJECT, $subjectTimer->result()); 131 | } 132 | if (!empty($this->subjectsGroupedByQueue)) { 133 | foreach ($this->subjectsGroupedByQueue as $queueName => $subjects) { 134 | $this->getApplyOperation()->createJob($subjects, $queueName, $this->getTripodOptions()); 135 | } 136 | $this->subjectsGroupedByQueue = array(); 137 | } 138 | } 139 | } 140 | } 141 | 142 | public function tearDown() 143 | { 144 | parent::tearDown(); 145 | $this->getStat()->increment(MONGO_QUEUE_DISCOVER_JOB . '.' . SUBJECT_COUNT, $this->subjectCount); 146 | } 147 | 148 | /** 149 | * Stat string for successful job timer 150 | * 151 | * @return string 152 | */ 153 | protected function getStatTimerSuccessKey() 154 | { 155 | return MONGO_QUEUE_DISCOVER_SUCCESS; 156 | } 157 | 158 | /** 159 | * Stat string for failed job increment 160 | * 161 | * @return string 162 | */ 163 | protected function getStatFailureIncrementKey() 164 | { 165 | return MONGO_QUEUE_DISCOVER_FAIL; 166 | } 167 | 168 | /** 169 | * @param array $data 170 | * @param string|null $queueName 171 | */ 172 | public function createJob(array $data, $queueName = null) 173 | { 174 | $configInstance = $this->getConfigInstance(); 175 | if (!$queueName) { 176 | $queueName = $configInstance::getDiscoverQueueName(); 177 | } elseif (strpos($queueName, $configInstance::getDiscoverQueueName()) === false) { 178 | $queueName = $configInstance::getDiscoverQueueName() . '::' . $queueName; 179 | } 180 | $this->submitJob($queueName, get_class($this), array_merge($data, $this->generateConfigJobArgs())); 181 | } 182 | 183 | /** 184 | * @param \Tripod\Mongo\ImpactedSubject $subject 185 | * @param string $queueName 186 | */ 187 | protected function addSubjectToQueue(\Tripod\Mongo\ImpactedSubject $subject, $queueName) 188 | { 189 | if (!array_key_exists($queueName, $this->subjectsGroupedByQueue)) { 190 | $this->subjectsGroupedByQueue[$queueName] = array(); 191 | } 192 | $this->subjectsGroupedByQueue[$queueName][] = $subject; 193 | } 194 | 195 | /** 196 | * For mocking 197 | * @return ApplyOperation 198 | */ 199 | protected function getApplyOperation() 200 | { 201 | if (!isset($this->applyOperation)) { 202 | $this->applyOperation = new ApplyOperation(); 203 | } 204 | return $this->applyOperation; 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/mongo/delegates/TransactionLog.class.php: -------------------------------------------------------------------------------- 1 | config = $config->getTransactionLogConfig(); 26 | $this->transaction_db = $config->getTransactionLogDatabase(); 27 | $this->transaction_collection = $this->transaction_db->selectCollection($this->config['collection']); 28 | } 29 | 30 | /** 31 | * @param string $transaction_id - the id you wish to assign to the new transaction 32 | * @param array $changes - an array serialization of the changeset to be applied 33 | * @param array $originalCBDs - an array of the serialized CBDs 34 | * @param string $storeName - the name of the database the changes are being applied to 35 | * @param string $podName - the name of the collection, in the database, the changes are being applied to 36 | * @throws \Tripod\Exceptions\Exception 37 | */ 38 | public function createNewTransaction($transaction_id, $changes, $originalCBDs, $storeName, $podName) 39 | { 40 | $transaction = array( 41 | "_id" => $transaction_id, 42 | "dbName"=>$storeName, 43 | "collectionName"=>$podName, 44 | "changes" => $changes, 45 | "status" => "in_progress", 46 | "startTime" => \Tripod\Mongo\DateUtil::getMongoDate(), 47 | "originalCBDs"=>$originalCBDs, 48 | "sessionId" => ((session_id() != '') ? session_id() : '') 49 | ); 50 | 51 | try { 52 | $result = $this->insertTransaction($transaction); 53 | if (!$result->isAcknowledged()) { 54 | throw new \Exception('Error creating new transaction'); 55 | } 56 | } catch(\Exception $e) { 57 | throw new \Tripod\Exceptions\Exception("Error creating new transaction: " . $e->getMessage()); 58 | } 59 | } 60 | 61 | /** 62 | * Updates the status of a transaction to cancelling. 63 | * If you passed in an Exception, the exception is logged in the transaction log. 64 | * 65 | * @param string $transaction_id the id of the transaction you wish to cancel 66 | * @param \Exception $error pass in the exception you wish to log 67 | */ 68 | public function cancelTransaction($transaction_id, \Exception $error=null) 69 | { 70 | $params = array('status' => 'cancelling'); 71 | if($error!=null) 72 | { 73 | $params['error'] = array('reason'=>$error->getMessage(), 'trace'=>$error->getTraceAsString()); 74 | } 75 | 76 | $this->updateTransaction( 77 | array("_id" => $transaction_id), 78 | array('$set' => $params), 79 | array("w" => 1, 'upsert'=>true) 80 | ); 81 | } 82 | 83 | /** 84 | * Updates the status of a transaction to failed, and adds a fail time. 85 | * If you passed in an Exception, the exception is logged in the transaction log 86 | * 87 | * @param string $transaction_id the id of the transaction you wish to set as failed 88 | * @param \Exception $error exception you wish to log 89 | */ 90 | public function failTransaction($transaction_id, \Exception $error=null) 91 | { 92 | $params = array('status' => 'failed', 'failedTime' => \Tripod\Mongo\DateUtil::getMongoDate()); 93 | if($error!=null) 94 | { 95 | $params['error'] = array('reason'=>$error->getMessage(), 'trace'=>$error->getTraceAsString()); 96 | } 97 | 98 | $this->updateTransaction( 99 | array("_id" => $transaction_id), 100 | array('$set' => $params), 101 | array('w' => 1, 'upsert'=>true) 102 | ); 103 | } 104 | 105 | /** 106 | * Update the status of a transaction to completed, and adds an end time 107 | * 108 | * @param string $transaction_id - the id of the transaction you want to mark as completed 109 | * @param array $newCBDs array of CBD's that represent the after state for each modified entity 110 | */ 111 | public function completeTransaction($transaction_id, Array $newCBDs) 112 | { 113 | 114 | $this->updateTransaction( 115 | array("_id" => $transaction_id), 116 | array('$set' => array('status' => 'completed', 'endTime' => \Tripod\Mongo\DateUtil::getMongoDate(), 'newCBDs'=>$newCBDs)), 117 | array('w' => 1) 118 | ); 119 | } 120 | 121 | /** 122 | * Retrieves a transaction from the transaction based on its id. The transaction is returned as an array 123 | * 124 | * @param string $transaction_id - the id of the transaction you wish to retrieve from the transaction log 125 | * @return Array representing the transaction document 126 | */ 127 | public function getTransaction($transaction_id) 128 | { 129 | return $this->transaction_collection->findOne(array("_id"=>$transaction_id)); 130 | 131 | } 132 | 133 | /** 134 | * Purges all transactions from the transaction log 135 | */ 136 | public function purgeAllTransactions() 137 | { 138 | $this->transaction_collection->drop(); 139 | } 140 | 141 | /** 142 | * @param string $storeName 143 | * @param string $podName 144 | * @param string|null $fromDate only transactions after this specified date will be replayed. This must be a datetime string i.e. '2010-01-15 00:00:00' 145 | * @param string|null $toDate only transactions after this specified date will be replayed. This must be a datetime string i.e. '2010-01-15 00:00:00' 146 | * @return Cursor 147 | * @throws \InvalidArgumentException 148 | */ 149 | public function getCompletedTransactions($storeName=null, $podName=null, $fromDate=null, $toDate=null) 150 | { 151 | $query = array(); 152 | $query['status'] = 'completed'; 153 | 154 | if(!empty($storeName) && !empty($podName)) 155 | { 156 | $query['dbName'] = $storeName; 157 | $query['collectionName'] = $podName; 158 | } 159 | 160 | if(!empty($fromDate)) { 161 | $q = array(); 162 | $q['$gte'] = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($fromDate)*1000); 163 | 164 | if(!empty($toDate)){ 165 | $q['$lte'] = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($toDate)*1000); 166 | } 167 | 168 | $query['endTime'] = $q; 169 | } 170 | 171 | return $this->transaction_collection->find($query, array('sort' => array('endTime'=>1))); 172 | } 173 | 174 | /** 175 | * @return int Total number of transactions in the transaction log 176 | */ 177 | public function getTotalTransactionCount() 178 | { 179 | return $this->transaction_collection->count(array()); 180 | } 181 | 182 | /** 183 | * @param string $storeName database name to filter on (optional) 184 | * @param string $podName collectionName to filter on (optional) 185 | * @return int Total number of completed transactions in the transaction log 186 | * @codeCoverageIgnore 187 | */ 188 | public function getCompletedTransactionCount($storeName=null, $podName=null) 189 | { 190 | if(!empty($storeName) && !empty($podName)) 191 | { 192 | return $this->transaction_collection->count(array('status'=>'completed','dbName'=>$storeName, 'collectionName'=>$podName)); 193 | } 194 | else 195 | { 196 | return $this->transaction_collection->count(array('status'=>'completed')); 197 | } 198 | } 199 | 200 | /* PROTECTED Functions */ 201 | 202 | /** 203 | * Proxy method to help with test mocking 204 | * @param array $transaction 205 | * @return InsertOneResult 206 | * @codeCoverageIgnore 207 | */ 208 | protected function insertTransaction($transaction) 209 | { 210 | return $this->transaction_collection->insertOne($transaction, array("w" => 1)); 211 | } 212 | 213 | /** 214 | * Proxy method to help with test mocking 215 | * @param array $query 216 | * @param array $update 217 | * @param array $options 218 | * @return UpdateOneResult 219 | * @codeCoverageIgnore 220 | */ 221 | protected function updateTransaction($query, $update, $options) 222 | { 223 | return $this->transaction_collection->updateOne($query, $update, $options); 224 | } 225 | 226 | 227 | } 228 | -------------------------------------------------------------------------------- /src/mongo/base/CompositeBase.class.php: -------------------------------------------------------------------------------- 1 | labeller->uri_to_alias($s); 31 | $subjectsToAlias[$s] = $resourceAlias; 32 | // build $filter for queries to impact index 33 | $filter[] = [_ID_RESOURCE=>$resourceAlias, _ID_CONTEXT=>$contextAlias]; 34 | } 35 | $query = [_ID_KEY => ['$in' => $filter]]; 36 | $docs = $this->getCollection()->find( 37 | $query, 38 | ['projection' => [_ID_KEY => true, 'rdf:type' => true]] 39 | ); 40 | 41 | $types = $this->getTypesInSpecifications(); 42 | 43 | if ($this->getCollection()->count($query) !== 0) { 44 | foreach ($docs as $doc) { 45 | $docResource = $doc[_ID_KEY][_ID_RESOURCE]; 46 | $docContext = $doc[_ID_KEY][_ID_CONTEXT]; 47 | $docHash = md5($docResource.$docContext); 48 | 49 | $docTypes = []; 50 | if (isset($doc['rdf:type'])) { 51 | if (isset($doc['rdf:type'][VALUE_URI])) { 52 | $docTypes[] = $doc['rdf:type'][VALUE_URI]; 53 | } else { 54 | foreach ($doc['rdf:type'] as $t) { 55 | if (isset($t[VALUE_URI])) { 56 | $docTypes[] = $t[VALUE_URI]; 57 | } 58 | } 59 | } 60 | } 61 | 62 | $currentSubjectProperties = []; 63 | if (isset($subjectsAndPredicatesOfChange[$docResource])) { 64 | $currentSubjectProperties = $subjectsAndPredicatesOfChange[$docResource]; 65 | } elseif (isset($subjectsToAlias[$docResource]) && 66 | isset($subjectsAndPredicatesOfChange[$subjectsToAlias[$docResource]])) { 67 | $currentSubjectProperties = $subjectsAndPredicatesOfChange[$subjectsToAlias[$docResource]]; 68 | } 69 | foreach ($docTypes as $type) { 70 | if ($this->checkIfTypeShouldTriggerOperation($type, $types, $currentSubjectProperties)) { 71 | if (!array_key_exists($this->getPodName(), $candidates)) { 72 | $candidates[$this->getPodName()] = []; 73 | } 74 | if (!array_key_exists($docHash, $candidates[$this->getPodName()])) { 75 | $candidates[$this->getPodName()][$docHash] = ['id'=>$doc[_ID_KEY]]; 76 | } 77 | } 78 | } 79 | } 80 | } 81 | 82 | // add to this any composites 83 | foreach ($this->findImpactedComposites($subjectsAndPredicatesOfChange, $contextAlias) as $doc) { 84 | $spec = $this->getSpecification($this->storeName, $doc[_ID_KEY]['type']); 85 | if (is_array($spec) && array_key_exists('from', $spec)) { 86 | if (!array_key_exists($spec['from'], $candidates)) { 87 | $candidates[$spec['from']] = []; 88 | } 89 | $docHash = md5($doc[_ID_KEY][_ID_RESOURCE] . $doc[_ID_KEY][_ID_CONTEXT]); 90 | 91 | if (!array_key_exists($docHash, $candidates[$spec['from']])) { 92 | $candidates[$spec['from']][$docHash] = [ 93 | 'id' => [ 94 | _ID_RESOURCE=>$doc[_ID_KEY][_ID_RESOURCE], 95 | _ID_CONTEXT=>$doc[_ID_KEY][_ID_CONTEXT], 96 | ] 97 | ]; 98 | } 99 | if (!array_key_exists('specTypes', $candidates[$spec['from']][$docHash])) { 100 | $candidates[$spec['from']][$docHash]['specTypes'] = []; 101 | } 102 | // Save the specification type so we only have to regen resources in that table type 103 | if (!in_array($doc[_ID_KEY][_ID_TYPE], $candidates[$spec['from']][$docHash]['specTypes'])) { 104 | $candidates[$spec['from']][$docHash]['specTypes'][] = $doc[_ID_KEY][_ID_TYPE]; 105 | } 106 | } 107 | } 108 | 109 | // convert operations to subjects 110 | $impactedSubjects = []; 111 | foreach (array_keys($candidates) as $podName) { 112 | foreach ($candidates[$podName] as $candidate) { 113 | $specTypes = (isset($candidate['specTypes']) ? $candidate['specTypes'] : []); 114 | $impactedSubjects[] = new \Tripod\Mongo\ImpactedSubject($candidate['id'], $this->getOperationType(), $this->getStoreName(), $podName, $specTypes); 115 | } 116 | } 117 | 118 | return $impactedSubjects; 119 | } 120 | 121 | /** 122 | * Returns an array of the rdf types that will trigger the specification 123 | * @return array 124 | */ 125 | abstract public function getTypesInSpecifications(); 126 | 127 | /** 128 | * @param array $resourcesAndPredicates 129 | * @param string $contextAlias 130 | * @return mixed // @todo: This may eventually return a either a Cursor or array 131 | */ 132 | abstract public function findImpactedComposites(array $resourcesAndPredicates, $contextAlias); 133 | 134 | /** 135 | * Returns the specification config 136 | * @param string $storeName 137 | * @param string $specId The specification id 138 | * @return array|null 139 | */ 140 | abstract public function getSpecification($storeName, $specId); 141 | 142 | /** 143 | * Test if the a particular type appears in the array of types associated with a particular spec and that the changeset 144 | * includes rdf:type (or is empty, meaning addition or deletion vs. update) 145 | * @param string $rdfType 146 | * @param array $validTypes 147 | * @param array $subjectPredicates 148 | * @return bool 149 | */ 150 | protected function checkIfTypeShouldTriggerOperation($rdfType, array $validTypes, array $subjectPredicates) 151 | { 152 | // We don't know if this is an alias or a fqURI, nor what is in the valid types, necessarily 153 | $types = [$rdfType]; 154 | try { 155 | $types[] = $this->labeller->qname_to_uri($rdfType); 156 | } catch (\Tripod\Exceptions\LabellerException $e) { 157 | // Not a qname, apparently 158 | } 159 | try { 160 | $types[] = $this->labeller->uri_to_alias($rdfType); 161 | } catch (\Tripod\Exceptions\LabellerException $e) { 162 | // Not a declared uri, apparently 163 | } 164 | 165 | $intersectingTypes = array_unique(array_intersect($types, $validTypes)); 166 | // If views have a matching type *at all*, the operation is triggered 167 | return (!empty($intersectingTypes)); 168 | } 169 | 170 | /** 171 | * For mocking 172 | * 173 | * @return \Tripod\Mongo\Jobs\ApplyOperation 174 | */ 175 | protected function getApplyOperation() 176 | { 177 | if (!isset($this->applyOperation)) { 178 | $this->applyOperation = new \Tripod\Mongo\Jobs\ApplyOperation(); 179 | } 180 | return $this->applyOperation; 181 | } 182 | 183 | /** 184 | * Queues a batch of ImpactedSubjects in a single ApplyOperation job 185 | * 186 | * @param \Tripod\Mongo\ImpactedSubject[] $subjects Array of ImpactedSubjects 187 | * @param string $queueName Queue name 188 | * @param array $jobOptions Job options 189 | * @return void 190 | */ 191 | protected function queueApplyJob(array $subjects, $queueName, array $jobOptions) 192 | { 193 | $this->getApplyOperation()->createJob($subjects, $queueName, $jobOptions); 194 | } 195 | 196 | /** 197 | * For mocking 198 | * 199 | * @param string $storeName 200 | * @return JobGroup 201 | */ 202 | protected function getJobGroup($storeName) 203 | { 204 | return new JobGroup($storeName); 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/tripod.inc.php: -------------------------------------------------------------------------------- 1 |