16 |
18 | 
19 |
20 |
21 |
19 |
20 |
23 |
26 | Intrigued? Learn more about Jina and checkout our Github! 27 |
28 |
29 | What just happened?
24 | This is Jina'saudio-to-audio-searchexample. It downloads AudioSet and indexes 1000 10-second audio clips via Jina search framework. We then randomly sample 5-second query audio clips extracted from the index audio clips as Queries, ask Jina to retrieve relevant results. Below is Jina's retrievals, where the left-most column is query audio. 25 |
26 | Intrigued? Learn more about Jina and checkout our Github! 27 |
30 |
Precision@{% TOP_K %}: {% PRECISION_EVALUATION %}
31 | 32 |33 | 34 |
Query | Top-K Results |
|---|
19 |
30 |
31 |
32 |
33 |
151 |
152 |
153 |
154 |
157 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
--------------------------------------------------------------------------------
/multires-lyrics-search/static/main.css:
--------------------------------------------------------------------------------
1 | .lyric-text {
2 | font-size: 10px;
3 | text-align: center;
4 | max-height: 30em;
5 | overflow: scroll;
6 | }
7 |
8 | .blockquote-footer {
9 | text-align: center;
10 | }
11 |
12 | .query-chunk {
13 | border-width: 1px;
14 | border-radius: 3px;
15 | border-style: solid;
16 | }
17 |
18 | .query-chunk-breakdown {
19 | margin: 5px;
20 | border-style: solid;
21 | border-width: 1px;
22 | border-radius: 5px;
23 | border-color: lightgray;
24 | padding: 5px;
25 | }
26 |
27 | .card {
28 | margin: 5px;
29 | }
--------------------------------------------------------------------------------
/multires-lyrics-search/static/vue-bindings.js:
--------------------------------------------------------------------------------
1 | var VueMasonryPlugin = window["vue-masonry-plugin"].VueMasonryPlugin;
2 | Vue.use(VueMasonryPlugin);
3 |
4 | const vm = new Vue({
5 | el: '#jina-ui',
6 | data: {
7 | serverUrl: 'http://localhost:45678/search',
8 | top_k: 50,
9 | topkDocs: [],
10 | topkDocsDict: {},
11 | results: [],
12 | searchQuery: '',
13 | queryChunks: [],
14 | selectQueryChunks: [],
15 | queryItem: [],
16 | docItem: null,
17 | loadedItem: 0,
18 | loadedQuery: 0,
19 | searchQueryIsDirty: false,
20 | isCalculating: false,
21 | distThreshold: 999,
22 | sliderOptions: {
23 | dotSize: 14,
24 | width: 'auto',
25 | height: 4,
26 | contained: false,
27 | direction: 'ltr',
28 | data: null,
29 | min: 999,
30 | max: 0,
31 | interval: 0.01,
32 | disabled: false,
33 | clickable: true,
34 | duration: 0.5,
35 | adsorb: false,
36 | lazy: false,
37 | tooltip: 'active',
38 | tooltipPlacement: 'top',
39 | tooltipFormatter: void 0,
40 | useKeyboard: false,
41 | keydownHook: null,
42 | dragOnClick: false,
43 | enableCross: true,
44 | fixed: false,
45 | minRange: void 0,
46 | maxRange: void 0,
47 | order: true,
48 | marks: false,
49 | dotOptions: void 0,
50 | process: true,
51 | dotStyle: void 0,
52 | railStyle: void 0,
53 | processStyle: void 0,
54 | tooltipStyle: void 0,
55 | stepStyle: void 0,
56 | stepActiveStyle: void 0,
57 | labelStyle: void 0,
58 | labelActiveStyle: void 0,
59 | }
60 | },
61 | mounted: function () {
62 |
63 | },
64 | components: {
65 | 'vueSlider': window['vue-slider-component'],
66 | },
67 | computed: {
68 | searchIndicator: function () {
69 | if (this.isCalculating) {
70 | return '⟳ Fetching new results...'
71 | } else if (this.searchQueryIsDirty) {
72 | return '... Typing'
73 | } else {
74 |
75 | return '✓ Done'
76 | }
77 | }
78 | },
79 | watch: {
80 | searchQuery: function () {
81 | this.searchQueryIsDirty = true
82 | this.expensiveOperation()
83 | },
84 | distThreshold: function () {
85 | this.refreshAllCards();
86 | }
87 | },
88 | methods: {
89 | clearAllSelect: function () {
90 | vm.queryChunks.forEach(function (item, i) {
91 | item['isSelect'] = !item['isSelect'];
92 | vm.refreshAllCards();
93 | });
94 | },
95 | selectChunk: function (item) {
96 | item['isSelect'] = !item['isSelect'];
97 | vm.refreshAllCards();
98 | },
99 | refreshAllCards: function () {
100 | vm.topkDocsDict = new Map(vm.topkDocs.map(i => [i.id, {
101 | 'text': i.text,
102 | 'hlchunk': [],
103 | 'renderHTML': i.text
104 | }]));
105 | vm.queryChunks.forEach(function (item, i) {
106 | if (!('isSelect' in item)) {
107 | item['isSelect'] = true;
108 | }
109 | if (item['isSelect']) {
110 | item.matches.forEach(function (r) {
111 | if (vm.topkDocsDict.has(r.parentId)) {
112 | let dist = r.scores['cosine'].value
113 | if (dist < vm.distThreshold) {
114 | // console.log(item)
115 | vm.topkDocsDict.get(r.parentId)['hlchunk'].push({
116 | 'range': r.location,
117 | 'idx': i,
118 | 'dist': dist,
119 | 'range_str': r.location[0] + ',' + r.location[1]
120 | });
121 | }
122 | if (dist < vm.sliderOptions.min) {
123 | vm.sliderOptions.min = dist.toFixed(2)
124 | }
125 | if (dist > vm.sliderOptions.max) {
126 | vm.sliderOptions.max = dist.toFixed(2)
127 | }
128 |
129 | } else {
130 | console.error(r.id);
131 | }
132 | });
133 | }
134 | });
135 | vm.topkDocsDict.forEach(function (value, key, map) {
136 | vm.topkDocsDict.get(key)['hlchunk'].sort(function (a, b) {
137 | return b['range'][0] - a['range'][0]
138 | })
139 | var replace_map = new Map();
140 | value['hlchunk'].forEach(function (item) {
141 | if (!replace_map.has(item['range_str'])) {
142 | replace_map.set(item['range_str'], [])
143 | }
144 | replace_map.get(item['range_str']).push(item)
145 |
146 | })
147 |
148 | replace_map.forEach(function (item, kk, mm) {
149 | value['renderHTML'] = replaceRange(value['renderHTML'], item[0]['range'][0], item[0]['range'][1], item)
150 | })
151 | })
152 | vm.$nextTick(function () {
153 | vm.$redrawVueMasonry('my-masonry');
154 | })
155 | },
156 | // This is where the debounce actually belongs.
157 | expensiveOperation: _.debounce(function () {
158 | this.isCalculating = true
159 | vm.selectQueryChunks.length = 0;
160 | $.ajax({
161 | url: this.serverUrl,
162 | type: "POST",
163 | contentType: "application/json",
164 | cache: false,
165 | data: JSON.stringify({
166 | "parameters": {"top_k": this.top_k},
167 | "data": [this.searchQuery]
168 | }),
169 | error: function (jqXHR, textStatus, errorThrown) {
170 | console.log(jqXHR);
171 | console.log(textStatus);
172 | console.log(errorThrown);
173 | },
174 | success: function (data) {
175 | vm.topkDocs = data.data.docs[0].matches;
176 | console.log('Number parents: ' + vm.topkDocs.length);
177 | vm.queryChunks = data.data.docs[0].chunks;
178 | console.log('Number chunks: ' + vm.queryChunks.length);
179 | vm.refreshAllCards();
180 | console.log('Success');
181 | },
182 | complete: function () {
183 | vm.isCalculating = false
184 | vm.searchQueryIsDirty = false
185 | vm.$nextTick(function () {
186 | vm.$redrawVueMasonry('my-masonry');
187 | })
188 | }
189 | });
190 |
191 | }, 500)
192 | }
193 | });
194 |
195 | function replaceRange(s, start, end, chunks) {
196 | var content = s.substring(start, end)
197 | chunks.forEach(function (c) {
198 | content = "" + content + ""
199 | })
200 | return s.substring(0, start) + content + s.substring(end);
201 | }
202 |
203 | function selectColor(number, colored) {
204 | if (!colored) {
205 | return `#fff`;
206 | }
207 | const hue = number * 137.508; // use golden angle approximation
208 | return `hsl(${hue},50%,75%)`;
209 | }
210 |
--------------------------------------------------------------------------------
/multires-lyrics-search/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jina-ai/examples/d8f903278597254bd96b1ed64fe8c9feefaa265c/multires-lyrics-search/tests/__init__.py
--------------------------------------------------------------------------------
/multires-lyrics-search/tests/conftest.py:
--------------------------------------------------------------------------------
1 | """Patch the birthday problem for random parts"""
2 |
3 | import pytest
4 |
5 |
6 | @pytest.fixture(scope='function', autouse=True)
7 | def patched_random_port(mocker):
8 | used_ports = set()
9 | from jina.helper import random_port
10 | from jina.excepts import NoAvailablePortError
11 |
12 | def _random_port():
13 |
14 | for i in range(10):
15 | _port = random_port()
16 |
17 | if _port is not None and _port not in used_ports:
18 | used_ports.add(_port)
19 | return _port
20 | raise NoAvailablePortError
21 |
22 | mocker.patch('jina.helper.random_port', new_callable=lambda: _random_port)
--------------------------------------------------------------------------------
/multires-lyrics-search/tests/requirements.txt:
--------------------------------------------------------------------------------
1 | click==8.0.1
2 | git+https://github.com/jina-ai/jina.git@v2.0.18#egg=jina[standard]
3 | pytest==6.1.2
4 | kaggle==1.5.12
5 | docker
6 | git+git://github.com/jina-ai/jina-commons@v0.0.3
--------------------------------------------------------------------------------
/multires-lyrics-search/tests/test_flow_integration.py:
--------------------------------------------------------------------------------
1 | __copyright__ = "Copyright (c) 2021 Jina AI Limited. All rights reserved."
2 | __license__ = "Apache-2.0"
3 |
4 | import os
5 | import shutil
6 | import glob
7 | from typing import List
8 | from click.testing import CliRunner
9 |
10 | import pytest
11 | from jina import Flow, Document
12 |
13 | from app import main
14 |
15 |
16 | def get_files_with_patterns(directory: str, match_patterns: List[str]) -> List[str]:
17 | """
18 | Returns all files from directory and subdirectories that match any of the patterns in the list.
19 | The returned list will only contain unique items.
20 |
21 | :param directory: Path to the directory
22 | :param match_patterns: A list of expressions to match the files against. E.g. `*.json`
23 | :return: List of matched files.
24 | """
25 | index_files = []
26 | for pattern in match_patterns:
27 | index_files += list(glob.glob(os.path.join(directory, '**', pattern), recursive=True))
28 | return list(set(index_files))
29 |
30 |
31 | @pytest.fixture(scope='session', autouse=True)
32 | def index(tmpdir_factory):
33 | """
34 | This fixtures runs automatically once before each test session.
35 | It indexes a small set of files into a test workspace and checks that the indexing
36 | completes correctly.
37 |
38 | Other tests can use the created workspace and test queries against it.
39 | """
40 | assert os.getcwd().endswith('multires-lyrics-search'), \
41 | "Please execute the tests from the root directory: >>> pytest tests/"
42 |
43 | workspace = os.path.join(tmpdir_factory.getbasetemp(), 'test-workspace')
44 | assert not os.path.isdir(workspace), 'Directory ./test-workspace exists. Please remove before testing'
45 | os.environ['JINA_WORKSPACE'] = workspace
46 | os.environ.setdefault('JINA_WORKSPACE_MOUNT',
47 | f'{os.environ.get("JINA_WORKSPACE")}:/workspace/workspace')
48 | os.environ.setdefault('JINA_PORT', str(45678))
49 |
50 | runner = CliRunner()
51 | result = runner.invoke(main, ['-t', 'index', '-n', '100'])
52 | assert result.stderr_bytes is None, f'Error messages found during indexing: {result.stderr}'
53 |
54 | assert os.path.isdir(workspace)
55 | index_files = get_files_with_patterns(workspace, ['*.bin', '*.lmdb', '*.lmdb-lock'])
56 | assert len(index_files) == 4, 'Expected three files in the workspace'
57 | for _file in index_files:
58 | assert os.path.getsize(_file) > 0, f'File {_file} is empty.'
59 |
60 | yield
61 | # shutil.rmtree(workspace) Not possible due to docker sudo rights
62 |
63 |
64 | def test_query_text(tmpdir_factory):
65 | def assert_result(response):
66 | docs = response.docs
67 | # check number of results
68 | assert len(docs) == 1
69 | assert len(docs[0].chunks) == 2
70 | parent_docs = docs[0].matches
71 | parent_ids = parent_docs.get_attributes('id')
72 | assert len(parent_docs) > 0
73 | for chunk in docs[0].chunks:
74 | assert len(chunk.matches) == 5 # top_k = 5
75 | match_ids = chunk.matches.get_attributes('id')
76 | assert len(match_ids) == len(list(set(match_ids)))
77 | for match in chunk.matches:
78 | assert match.text is not None
79 | assert match.location is not None
80 | assert match.parent_id in parent_ids
81 | assert match.text in parent_docs[parent_ids.index(match.parent_id)].text
82 |
83 | flow = Flow.load_config('flows/query.yml')
84 | with flow:
85 | search_text = 'looked through every window then. hello world.'
86 | doc = Document(content=search_text, mime_type='text/plain')
87 | response = flow.post('/search', inputs=doc, parameters={'top_k': 5}, return_results=True)
88 | assert_result(response[0])
89 |
--------------------------------------------------------------------------------
/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | norecursedirs = cross-modal-search/img_emb/* cross-modal-search/txt_emb/* openapi/python-flask/openapi_server/*
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/.github/images/storage.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/.gitignore:
--------------------------------------------------------------------------------
1 | workspace*
2 | env
3 | results
4 | __pycache__
5 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/README.md:
--------------------------------------------------------------------------------
1 | # Querying While Indexing in the Wikipedia Search Example
2 |
3 | | About this example: | |
4 | | ------------- | ------------- |
5 | | Learnings | How to configure Jina for querying while indexing |
6 | | Used for indexing | Text data |
7 | | Used for querying | Text data |
8 | | Dataset used | [Wikipedia dataset from kaggle](https://www.kaggle.com/mikeortman/wikipedia-sentences) |
9 | | Model used | [flair-text](https://github.com/flairNLP/flair) |
10 |
11 | This is an example of using [Jina](http://www.jina.ai) to support both querying and indexing simultaneously in our [Wikipedia sentence search example](https://github.com/jina-ai/examples/tree/master/wikipedia-sentences).
12 |
13 | ## Table of contents:
14 |
15 | * [Prerequisites](#prerequisites)
16 | * [What is querying while indexing?](#what-is-querying-while-indexing)
17 | * [Configuration changes](#configuration-changes)
18 | * [🐍 Build the app with Python](#-build-the-app-with-python)
19 | * [Flow diagrams](#flow-diagrams)
20 | * [🔮 Overview of the files](#-overview-of-the-files)
21 | * [Troubleshooting](#troubleshooting)
22 | * [⏭️ Next steps](#-next-steps)
23 | * [👩👩👧👦 Community](#-community)
24 | * [🦄 License](#-license)
25 |
26 | ## Prerequisites
27 |
28 | - Run and understand our [Wikipedia sentence search example](https://github.com/jina-ai/examples/tree/master/wikipedia-sentences)
29 |
30 | ## What is querying while indexing?
31 |
32 | Querying while indexing means you are able to still query your data while new data is simultaneously being inserted (or updated, or deleted).
33 | Jina achieves this with its dump-reload feature.
34 |
35 | ## Configuration changes
36 |
37 | This feature requires you to split the Flow, one for Indexing (and Updates, Deletes) and one for Querying, and have them running at the same time.
38 | Also, you will need to replace the indexers in Flows.
39 | The Index Flow (also referred to as the Storage Flow) will require a [Storage Indexer](https://github.com/jina-ai/executors/tree/main/jinahub/indexers/storage), while the Query Flow requires a [Compound Searcher](https://github.com/jina-ai/executors/tree/main/jinahub/indexers/searcher).
40 |
41 | In our case we use :
42 |
43 | - [LMDBStorage](https://github.com/jina-ai/executors/tree/main/jinahub/indexers/storage/LMDBStorage), which uses a disk-based key-value storage [LMDB](https://lmdb.readthedocs.io/) as a storage engine.
44 | - [FaissLMDBSearcher](https://github.com/jina-ai/executors/tree/main/jinahub/indexers/searcher/compound/FaissLMDBSearcher), which uses the [`faiss`](https://github.com/spotify/annoy) algorithm to provide faster query results and LMDB to retrieve the metadata.
45 |
46 | _____
47 |
48 | ## 🐍 Build the app with Python
49 |
50 | These instructions explain how to run the example yourself and deploy it with Python.
51 |
52 | ### 🗝️ Requirements
53 |
54 | 1. Have a working Python 3.7 or 3.8 environment.
55 | 1. We recommend creating a [new Python virtual environment](https://docs.python.org/3/tutorial/venv.html) to have a clean installation of Jina and prevent dependency conflicts.
56 | 1. Install [Docker Engine](https://docs.docker.com/engine/install/).
57 | 1. Have at least 5 GB of free space on your hard drive.
58 |
59 |
60 | ### Running the example
61 |
62 | ### 👾 Step 1. Clone the repo and install Jina
63 |
64 | Begin by cloning the repo so you can get the required files and datasets. (If you already have the examples repository on your machine make sure to fetch the most recent version)
65 |
66 | ```sh
67 | git clone https://github.com/jina-ai/examples
68 | cd examples/wikipedia-sentences-query-while-indexing
69 | ```
70 |
71 | Let's install `jina` and the other required libraries. For further information on installing jina check out [our documentation](https://docs.jina.ai/get-started/install/).
72 |
73 | ```sh
74 | pip install -r requirements.txt
75 | ```
76 |
77 | In order to run the example you will need to do the following:
78 |
79 | ### 📥 Step 2. Download your data to search (Optional)
80 |
81 | The repo includes a small subset of the Wikipedia dataset, for quick testing. You can just use that.
82 |
83 | If you want to use the entire dataset, run `bash get_data.sh` and then modify the `DATA_FILE` constant (in `app.py`) to point to that file.
84 |
85 | ### 🏃 Step 3. Running the Flows
86 |
87 | In this example, we use [JinaD]((https://docs.jina.ai/advanced/daemon/#remote-management-via-jinad)) to serve the two Flows (Index and Query) and listen to incoming requests.
88 |
89 | 1. Start `JinaD` server using the below command.
90 |
91 | ```bash
92 | docker run --add-host host.docker.internal:host-gateway \
93 | -v /var/run/docker.sock:/var/run/docker.sock \
94 | -v /tmp/jinad:/tmp/jinad \
95 | -p 8000:8000 \
96 | --name jinad \
97 | -d jinaai/jina:2.1.0-daemon
98 | ```
99 |
100 | 2. Run `python app.py -t flows`
101 |
102 | This will create the two Flows, and then repeatedly do the following (which can also be done in any other REST client), every 10 seconds:
103 |
104 | 1. Index 5 Documents.
105 | 2. Send a `DUMP` request to the Storage (Index) Flow to dump its data to a specific location.
106 | 3. Send a `ROLLING_UPDATE` request to the Query Flow to take down its Indexers and start them again, with the new data located at the respective path.
107 |
108 | **Warning**: If you want to use the entire wikipedia dataset, run `bash get_data.sh` and then modify the `DATA_FILE` constant to point to that file.
109 |
110 | ### 🔎 Step 4: Query your data
111 |
112 | Finally, in a second terminal, run `python app.py -t client`
113 |
114 | This will prompt you for a query, send the query to the Query Flow, and then show you the results.
115 |
116 | Since the Flows uses `http` protocol, you can query the REST API with whatever `Client` provided within jina or use `cURL`, `Postman` or [custom Swagger UI provided with jina](https://docs.jina.ai/fundamentals/practice-your-learning/#query-via-swaggerui) etc.
117 |
118 | #### Cleanup
119 |
120 | JinaD creates several containers during this process. In order to remove all the containers do the following after you are done using the example:
121 |
122 | `docker stop $(docker ps -a -q)`
123 | and
124 | `docker rm $(docker ps -a -q)`
125 |
126 | ## Flow diagrams
127 |
128 | Below you can see a graphical representation of the Flow pipeline:
129 |
130 | #### Storage Flow
131 |
132 | 
133 |
134 | #### Query Flow
135 |
136 | 
137 |
138 | Notice the following:
139 |
140 | - the encoder has the same configuration
141 | - the Query Flow uses replicas. One replica continues to serve requests while the other is being reloaded.
142 | - the Indexer in the Query Flow is actually made up of two Indexers: one for vectors, one for Document metadata. On the Storage Flow, this data is stored into one Storage Indexer.
143 |
144 | ## 🔮 Overview of the files
145 |
146 | | File or folder | Contents |
147 | | -------------------- | ---------------------------------------------------------------------------------------------------------------- |
148 | | 📂 `data/` | Folder where the data files are stored |
149 | | 📂 `flows/` | Folder to store Flow configuration |
150 | | --- 📃 `storage.yml` | YAML file to configure Storage (Index) Flow |
151 | | --- 📃 `query.yml` | YAML file to configure Querying Flow |
152 | | 🐍 `app.py` | Code file for the example |
153 |
154 | _________
155 |
156 | ## ⏭️ Next steps
157 |
158 | Did you like this example and are you interested in building your own? For a detailed tutorial on how to build your Jina app check out [How to Build Your First Jina App](https://docs.jina.ai/chapters/my_first_jina_app/#how-to-build-your-first-jina-app) guide in our documentation.
159 |
160 | If you have any issues following this guide, you can always get support from our [Slack community](https://slack.jina.ai) .
161 |
162 | ## 👩👩👧👦 Community
163 |
164 | - [Slack channel](https://slack.jina.ai) - a communication platform for developers to discuss Jina.
165 | - [LinkedIn](https://www.linkedin.com/company/jinaai/) - get to know Jina AI as a company and find job opportunities.
166 | - [](https://twitter.com/JinaAI_) - follow us and interact with us using hashtag `#JinaSearch`.
167 | - [Company](https://jina.ai) - know more about our company. We are fully committed to open-source!
168 |
169 | ## 🦄 License
170 |
171 | Copyright (c) 2021 Jina AI Limited. All rights reserved.
172 |
173 | Jina is licensed under the Apache License, Version 2.0. See LICENSE for the full license text.
174 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jina-ai/examples/d8f903278597254bd96b1ed64fe8c9feefaa265c/wikipedia-sentences-query-while-indexing/__init__.py
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/app.py:
--------------------------------------------------------------------------------
1 | __copyright__ = "Copyright (c) 2021 Jina AI Limited. All rights reserved."
2 | __license__ = "Apache-2.0"
3 |
4 | import os
5 | import time
6 | import traceback
7 | from typing import List, Dict
8 |
9 | import click
10 | from daemon.clients import JinaDClient
11 | from jina.logging.logger import JinaLogger
12 | from jina import __default_host__, Document, DocumentArray, Client
13 |
14 | os.environ['JINA_LOG_LEVEL'] = 'DEBUG'
15 |
16 | HOST = __default_host__ # change this if you are using remote jinad
17 | JINAD_PORT = 8000 # change this if you start jinad on a different port
18 | DUMP_PATH = '/jinad_workspace/dump' # the path where to dump
19 | SHARDS = 1 # change this if you change pods/query_indexer.yml
20 | DUMP_RELOAD_INTERVAL = 10 # time between dump - rolling update calls
21 | DATA_FILE = 'data/toy.txt' # change this if you get the full data
22 | DOCS_PER_ROUND = 5 # nr of documents to index in each round
23 | STORAGE_FLOW_YAML_FILE = 'storage.yml' # indexing Flow yaml name
24 | QUERY_FLOW_YAML_FILE = 'query.yml' # querying Flow yaml name
25 | STORAGE_REST_PORT = 9000 # REST port of storage Flow, defined in flows/storage.yml
26 | QUERY_REST_PORT = 9001 # REST port of Query Flow, defined in flows/query.yml
27 |
28 | logger = JinaLogger('jina')
29 | cur_dir = os.path.dirname(os.path.abspath(__file__))
30 | jinad_client = JinaDClient(host=HOST, port=JINAD_PORT, timeout=10 * 60)
31 |
32 |
33 | def docarray_from_file(filename):
34 | docs = []
35 | with open(filename) as f:
36 | for line in f:
37 | docs.append(Document(text=line))
38 | return DocumentArray(docs)
39 |
40 |
41 | def query_restful():
42 | while True:
43 | text = input('please type a sentence: ')
44 | if not text:
45 | break
46 |
47 | query_doc = Document()
48 | query_doc.text = text
49 | response = query_docs(query_doc)
50 | matches = response[0].data.docs[0].matches
51 | len_matches = len(matches)
52 | logger.info(f'Ta-Dah🔮, {len_matches} matches we found for: "{text}" :')
53 |
54 | for idx, match in enumerate(matches):
55 | score = match.scores['euclidean'].value
56 | if score < 0.0:
57 | continue
58 | logger.info(f'> {idx:>2d}({score:.2f}). {match.text}')
59 |
60 |
61 | def index_docs(docs: List[Dict], round: int):
62 | docs_to_send = docs[round * DOCS_PER_ROUND : (round + 1) * DOCS_PER_ROUND]
63 | logger.info(f'Indexing {len(docs_to_send)} document(s)...')
64 | Client(host=HOST, port=STORAGE_REST_PORT, protocol='http').index(inputs=docs_to_send)
65 |
66 |
67 | def query_docs(docs: Document):
68 | logger.info(f'Searching document {docs}...')
69 | return Client(host=HOST, port=QUERY_REST_PORT, protocol='http').search(inputs=docs, return_results=True)
70 |
71 |
72 | def create_flows():
73 | workspace_id = jinad_client.workspaces.create(paths=[os.path.join(cur_dir, 'flows')])
74 | jinad_workspace = jinad_client.workspaces.get(workspace_id)['metadata']['workdir']
75 |
76 | logger.info('Creating storage Flow...')
77 | storage_flow_id = jinad_client.flows.create(
78 | workspace_id=workspace_id, filename=STORAGE_FLOW_YAML_FILE, envs={'JINAD_WORKSPACE': jinad_workspace}
79 | )
80 | logger.info(f'Created successfully. Flow ID: {storage_flow_id}')
81 | logger.info('Creating Query Flow...')
82 | query_flow_id = jinad_client.flows.create(
83 | workspace_id=workspace_id, filename=QUERY_FLOW_YAML_FILE, envs={'JINAD_WORKSPACE': jinad_workspace}
84 | )
85 | logger.info(f'Created successfully. Flow ID: {query_flow_id}')
86 | return storage_flow_id, query_flow_id, workspace_id
87 |
88 |
89 | def dump_and_roll_update(storage_flow_id: str, query_flow_id: str):
90 | docs = docarray_from_file(DATA_FILE)
91 | logger.info(f'starting dump and rolling-update process')
92 | round = 0
93 | while True:
94 | logger.info(f'round {round}:')
95 | index_docs(docs, round)
96 | current_dump_path = os.path.join(DUMP_PATH, str(round))
97 |
98 | logger.info(f'dumping...')
99 | Client(host=HOST, port=STORAGE_REST_PORT, protocol='http').post(
100 | on='/dump',
101 | parameters={'shards': SHARDS, 'dump_path': current_dump_path},
102 | target_peapod='storage_indexer',
103 | )
104 |
105 | # JinaD is used for ctrl requests on Flows
106 | logger.info(f'performing rolling update across replicas...')
107 | jinad_client.flows.update(
108 | id=query_flow_id,
109 | kind='rolling_update',
110 | pod_name='query_indexer',
111 | dump_path=current_dump_path,
112 | )
113 | logger.info(f'rolling update done. sleeping for {DUMP_RELOAD_INTERVAL}secs...')
114 | time.sleep(DUMP_RELOAD_INTERVAL)
115 | round += 1
116 |
117 |
118 | def cleanup(storage_flow_id, query_flow_id, workspace_id):
119 | jinad_client.flows.delete(storage_flow_id)
120 | jinad_client.flows.delete(query_flow_id)
121 | jinad_client.workspaces.delete(workspace_id)
122 |
123 |
124 | @click.command()
125 | @click.option(
126 | '--task',
127 | '-t',
128 | type=click.Choice(['flows', 'client'], case_sensitive=False),
129 | )
130 | def main(task: str):
131 | """main entrypoint for this example"""
132 | if task == 'flows':
133 | # start a Index Flow, dump the data from the Index Flow, and load it into the Query Flow.
134 | try:
135 | storage_flow_id, query_flow_id, workspace_id = create_flows()
136 | # starting a loop that
137 | # - indexes some data in batches
138 | # - sends request to storage Flow in JinaD to dump its data to a location
139 | # - send request to Query Flow in JinaD to perform rolling update across its replicas,
140 | # which reads the new data in the dump
141 | dump_and_roll_update(storage_flow_id, query_flow_id)
142 | except (Exception, KeyboardInterrupt) as e:
143 | if e:
144 | logger.warning(f'Caught: {e}. Original stacktrace following:')
145 | logger.error(traceback.format_exc())
146 | logger.info('Shutting down and cleaning Flows in JinaD...')
147 | cleanup(storage_flow_id, query_flow_id, workspace_id)
148 |
149 | elif task == 'client':
150 | query_restful()
151 |
152 |
153 | if __name__ == '__main__':
154 | main()
155 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/flows/query.yml:
--------------------------------------------------------------------------------
1 | jtype: Flow # we define the search Flow
2 | version: '1'
3 | with:
4 | protocol: http # we use the REST API
5 | port_expose: 9001 # the port the Flow will listen on
6 | executors: # the list of components in this Flow
7 | - name: query_encoder # the name of this executor. This one takes the text and transforms it into vectors to be used in searching
8 | uses: jinahub+docker://FlairTextEncoder # we use a pre-built Executor docker image
9 | timeout_ready: -1 # disable timing out. (downloading the image can take some time)
10 | - name: query_indexer # the name. This is a Compound Executor, formed of a vector searcher and a key-value db
11 | uses: jinahub+docker://FaissLMDBSearcher # again, the docker image
12 | replicas: 2 # we want to replicate this executor, for better performance. This creates two identical copies. Requests are passed to either one
13 | timeout_ready: -1 # disable timing out. (downloading the image can take some time)
14 | volumes: $JINAD_WORKSPACE:/jinad_workspace # we need a workspace where the LMDB db file will be stored
15 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/flows/storage.yml:
--------------------------------------------------------------------------------
1 | jtype: Flow # we define the Flow used for storing (CRUD operations)
2 | version: '1'
3 | with:
4 | protocol: http # we want to use the REST HTTP API
5 | port_expose: 9000 # the port to listen on. This is referenced in `app.py`
6 | executors: # the components in this Flow
7 | - name: storage_encoder # the name. This is the Encoder (transforms the text into vectors)
8 | uses: jinahub+docker://FlairTextEncoder # we use a pre-built Executor from Jina Hub
9 | timeout_ready: -1 # disable timing out on startup (downloading image can take some time)
10 | - name: storage_indexer # the name. This stores the data in an LMDB db
11 | uses: jinahub+docker://LMDBStorage # again, we use a docker image
12 | timeout_ready: -1 # disable startup
13 | volumes: $JINAD_WORKSPACE:/jinad_workspace # workspace where the file will be stored
14 |
15 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/get_data.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | DATASET="mikeortman/wikipedia-sentences"
3 | DATA_DIR="data"
4 | LINES=3000
5 |
6 | cd ${DATA_DIR}
7 | kaggle datasets download -d ${DATASET}
8 | unzip wikipedia-sentences.zip
9 | rm -f toy-data.txt
10 | rm -f wikipedia-sentences.zip
11 | mv wikisent2.txt input.txt
12 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/manifest.yml:
--------------------------------------------------------------------------------
1 | manifest_version: 1
2 | name: wikipedia-sentences-30k-query-while-indexing
3 | description: 'Example Jina app for searching 30,000 sentences from Wikipedia'
4 | author: Cristian Mitroi (cristian.mitroi@jina.ai)
5 | url: https://github.com/jina-ai/examples/tree/master/wikipedia-sentences-query-while-indexing
6 | vendor: Jina AI Limited
7 | documentation: https://github.com/jina-ai/examples/tree/master/wikipedia-sentences-query-while-indexing
8 | version: 0.1
9 | license: apache-2.0
10 | keywords: [NLP, wikipedia, text, distilbert, example, transformers]
11 | type: app
12 | kind: example
13 | avatar: None
14 | platform: "linux/amd64"
15 | update: "None"
16 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/requirements.txt:
--------------------------------------------------------------------------------
1 | jina[daemon]==2.1.0
2 | kaggle==1.5.12
3 | click==7.1.2
4 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jina-ai/examples/d8f903278597254bd96b1ed64fe8c9feefaa265c/wikipedia-sentences-query-while-indexing/tests/__init__.py
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/tests/requirements.txt:
--------------------------------------------------------------------------------
1 | git+https://github.com/jina-ai/jina.git@v2.1.0#egg=jina[daemon]
2 | click==7.1.2
3 |
--------------------------------------------------------------------------------
/wikipedia-sentences-query-while-indexing/tests/test_query_while_indexing.py:
--------------------------------------------------------------------------------
1 | import time
2 | from threading import Thread
3 |
4 | from jina import Document, __default_host__, Client
5 | from daemon.clients import JinaDClient
6 | from jina.logging.logger import JinaLogger
7 |
8 | HOST = __default_host__
9 | JINAD_PORT = 8000
10 | QUERY_REST_PORT = 9001
11 | logger = JinaLogger('test')
12 |
13 |
14 | def query_docs(docs: Document):
15 | logger.info(f'Searching document {docs}...')
16 | return Client(host=HOST, port=QUERY_REST_PORT, protocol='http').search(inputs=docs, return_results=True)
17 |
18 |
19 | def test_query_while_indexing():
20 | try:
21 | from app import create_flows, dump_and_roll_update
22 |
23 | jinad_client = JinaDClient(host=HOST, port=JINAD_PORT)
24 | assert jinad_client.alive, 'cannot reach jinad'
25 |
26 | storage_flow_id, query_flow_id, workspace_id = create_flows()
27 | # start rolling update in the background
28 | Thread(target=dump_and_roll_update, args=(storage_flow_id, query_flow_id), daemon=True).start()
29 |
30 | logger.info('sleeping for 30 secs to allow 1 round of index, dump & rolling update')
31 | time.sleep(30)
32 | query_doc = Document(text='hello world')
33 | response = query_docs(query_doc)
34 | matches = response[0].data.docs[0].matches
35 | logger.info(f'got {len(matches)} matches')
36 | assert matches
37 |
38 | except (Exception, KeyboardInterrupt):
39 | raise
40 |
41 | finally:
42 | from app import cleanup
43 |
44 | cleanup(storage_flow_id, query_flow_id, workspace_id)
45 |
--------------------------------------------------------------------------------
/wikipedia-sentences/.dockerignore:
--------------------------------------------------------------------------------
1 | .dockerignore
2 | .git
3 | .github
4 | .gitignore
5 | data
6 | env
7 | get-data.sh
8 | tests
9 |
--------------------------------------------------------------------------------
/wikipedia-sentences/.github/flow.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jina-ai/examples/d8f903278597254bd96b1ed64fe8c9feefaa265c/wikipedia-sentences/.github/flow.png
--------------------------------------------------------------------------------
/wikipedia-sentences/.gitignore:
--------------------------------------------------------------------------------
1 | workspace*
2 | env
3 | results
4 | __pycache__
5 |
--------------------------------------------------------------------------------
/wikipedia-sentences/README.md:
--------------------------------------------------------------------------------
1 | # Semantic Wikipedia Search with Transformers and DistilBERT
2 |
3 | 
4 |
5 | ## Table of contents:
6 |
7 | - [Overview](#overview)
8 | - [🐍 Build the app with Python](#-build-the-app-with-python)
9 | - [🔮 Overview of the files in this example](#-overview-of-the-files-in-this-example)
10 | - [🌀 Flow diagram](#-flow-diagram)
11 | - [🔨 Next steps, building your own app](#-next-steps-building-your-own-app)
12 | - [🙍 Community](#-community)
13 | - [🦄 License](#-license)
14 |
15 | ## Overview
16 | | | |
17 | | ------------- | ------------- |
18 | | Summary | This showcases a semantic text search app |
19 | | Data for indexing | Wikipedia corpus |
20 | | Data for querying | A text sentence |
21 | | Dataset used | [Kaggle Wikipedia corpus](kaggle.com/mikeortman/wikipedia-sentences) |
22 | | ML model used | [`distilbert-base-nli-stsb-mean-tokens `](https://huggingface.co/sentence-transformers/distilbert-base-nli-stsb-mean-tokens) |
23 |
24 | This example shows you how to build a simple semantic search app powered by [Jina](http://www.jina.ai)'s neural search framework. You can index and search text sentences from Wikipedia using a state-of-the-art machine learning [`distilbert-base-nli-stsb-mean-tokens `](https://huggingface.co/sentence-transformers/distilbert-base-nli-stsb-mean-tokens) language model from the [Transformers](https://huggingface.co) library.
25 |
26 | | item | content |
27 | |--------|--------------------------------------------------|
28 | | Input | 1 text file with 1 sentence per line |
29 | | Output | *top_k* number of sentences that match input query |
30 |
31 | ## 🐍 Build the app with Python
32 |
33 | These instructions explain how to build the example yourself and deploy it with Python. If you want to skip the building steps and just run the app, check out the [Docker section](#---deploy-the-prebuild-application-using-docker) below.
34 |
35 |
36 | ### 🗝️ Requirements
37 | 1. You have a working Python 3.7 or 3.8 environment.
38 | 2. We recommend creating a [new Python virtual environment](https://docs.python.org/3/tutorial/venv.html) to have a clean installation of Jina and prevent dependency conflicts.
39 | 3. You have at least 2 GB of free space on your hard drive.
40 |
41 | ### 👾 Step 1. Clone the repo and install Jina
42 |
43 |
44 | Begin by cloning the repo, so you can get the required files and datasets. In case you already have the examples repository on your machine make sure to fetch the most recent version.
45 |
46 | ```sh
47 | git clone https://github.com/jina-ai/examples
48 | cd examples/wikipedia-sentences
49 | ```
50 |
51 | In your terminal, you should now be located in you the wikipedia-sentences folder. Let's install Jina and the other required Python libraries. For further information on installing Jina check out our [documentation](https://docs.jina.ai/chapters/core/setup/).
52 |
53 |
54 | ```sh
55 | pip install -r requirements.txt
56 | ```
57 | If this command runs without any error messages, you can then move onto step two.
58 |
59 | ### 📥 Step 2. Download your data to search
60 |
61 | By default, a small test dataset is used for indexing. This can lead to bad search results.
62 |
63 | To index the [full dataset](https://www.kaggle.com/mikeortman/wikipedia-sentences) (around 900 MB):
64 |
65 | 1. Set up [Kaggle](https://www.kaggle.com/docs/api#getting-started-installation-&-authentication)
66 | 2. Run the script: `sh get_data.sh`
67 | 3. Index your new dataset: `python app.py -t index -d full -n $num_docs`
68 |
69 | The whole dataset contains about 8 Million wikipedia sentences, indexing all of this will take a very long time.
70 | Therefore, we recommend selecting only a subset of the data, the number of elements can be selected by the `-n` flag.
71 | We recommend values smaller than 100000. For larger indexes, the SimpleIndexer used in this example will be very slow also in query time.
72 | It is then recommended to use more advanced indexers like the FaissIndexer.
73 |
74 | ### 🏃 Step 3. Index your data
75 |
76 | Index your data by running:
77 |
78 | ```sh
79 | python app.py -t index
80 | ```
81 | Here, we can also specify the number of documents to index with ```--num_docs``` / ```-n``` (defult is 10000).
82 |
83 | ### 🔎 Step 4. Query your indexed data
84 |
85 | A search prompt will appear in your terminal after running:
86 |
87 | ```sh
88 | python app.py -t query
89 | ```
90 |
91 | See the text below for an example search query and response.
92 | You can also specify the top k search results with ```--top_k``` / ```-k``` (default is 5)
93 |
94 | ```
95 | please type a sentence: What is ROMEO
96 |
97 | Ta-Dah🔮, here are what we found for: What is ROMEO
98 | > 0(0.36). The ROMEO website, iOS app and Android app are commonly used by the male gay community to find friends, dates, love or get informed about LGBT+ topics.
99 |
100 | ```
101 |
102 | ## 🔮 Overview of the files in this example
103 | Here is a small overview if you're interested in understanding what each file in this example is doing.
104 |
105 | | File | Explanation |
106 | |---|---|
107 | |📂 `test/*` | Various maintenance tests to keep the example running. |
108 | |📃 `app.py` | The gateway code to that runs the index & query Flow. |
109 | |📃 `get_data.sh` | Downloads the Kaggle dataset. |
110 | |📃 `requirements.txt` | Contains all required python libraries. |
111 |
112 |
113 | ## 🌀 Flow diagram
114 |
115 | This diagram provides a visual representation of the flow in this example, showing which Executors are used in which order:
116 |
117 | 
118 |
119 | It can be seen that the flow for this example is quite simple. We receive input Documents from the gateway,
120 | which are then fed into a transformer. This transformer computes an embedding based on the text of the document.
121 | Then, the documents are sent to the indexer which does the following:
122 | - Index time: Store all the documents on disk (in the workspace folder).
123 | - Query time: Compare the query document embedding with all stored embeddings and return closest matches
124 |
125 | ## ⏭️ Next steps, building your own app
126 |
127 | Did you like this example and are you interested in building your own? For a detailed tuturial on how to build your Jina app check out [How to Build Your First Jina App](https://docs.jina.ai/chapters/my_first_jina_app/#how-to-build-your-first-jina-app) guide in our documentation.
128 |
129 | - [Enable querying while indexing](https://github.com/jina-ai/examples/tree/master/wikipedia-sentences-query-while-indexing)
130 |
131 | ## 👩👩👧👦 Community
132 |
133 | - [Slack channel](https://slack.jina.ai) - a communication platform for developers to discuss Jina
134 | - [LinkedIn](https://www.linkedin.com/company/jinaai/) - get to know Jina AI as a company and find job opportunities
135 | - [](https://twitter.com/JinaAI_) - follow us and interact with us using hashtag `#JinaSearch`
136 | - [Company](https://jina.ai) - know more about our company, we are fully committed to open-source!
137 |
138 | ## 🦄 License
139 |
140 | Copyright (c) 2021 Jina AI Limited. All rights reserved.
141 |
142 | Jina is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/jina-ai/examples/blob/master/LICENSE) for the full license text.
143 |
--------------------------------------------------------------------------------
/wikipedia-sentences/app.py:
--------------------------------------------------------------------------------
1 | __copyright__ = "Copyright (c) 2021 Jina AI Limited. All rights reserved."
2 | __license__ = "Apache-2.0"
3 |
4 | import os
5 | import sys
6 | import click
7 | import random
8 | from jina import Flow, Document, DocumentArray
9 | from jina.logging.predefined import default_logger as logger
10 |
11 | MAX_DOCS = int(os.environ.get('JINA_MAX_DOCS', 10000))
12 |
13 |
14 | def config(dataset: str):
15 | if dataset == 'toy':
16 | os.environ['JINA_DATA_FILE'] = os.environ.get('JINA_DATA_FILE', 'data/toy-input.txt')
17 | elif dataset == 'full':
18 | os.environ['JINA_DATA_FILE'] = os.environ.get('JINA_DATA_FILE', 'data/input.txt')
19 | os.environ['JINA_PORT'] = os.environ.get('JINA_PORT', str(45678))
20 | cur_dir = os.path.dirname(os.path.abspath(__file__))
21 | os.environ.setdefault('JINA_WORKSPACE', os.path.join(cur_dir, 'workspace'))
22 | os.environ.setdefault('JINA_WORKSPACE_MOUNT',
23 | f'{os.environ.get("JINA_WORKSPACE")}:/workspace/workspace')
24 |
25 |
26 | def print_topk(resp, sentence):
27 | for doc in resp.data.docs:
28 | print(f"\n\n\nTa-Dah🔮, here's what we found for: {sentence}")
29 | for idx, match in enumerate(doc.matches):
30 | score = match.scores['cosine'].value
31 | print(f'> {idx:>2d}({score:.2f}). {match.text}')
32 |
33 |
34 | def input_generator(num_docs: int, file_path: str):
35 | with open(file_path) as file:
36 | lines = file.readlines()
37 | num_lines = len(lines)
38 | random.shuffle(lines)
39 | for i in range(min(num_docs, num_lines)):
40 | yield Document(text=lines[i])
41 |
42 |
43 | def index(num_docs):
44 | flow = Flow().load_config('flows/flow.yml')
45 | data_path = os.path.join(os.path.dirname(__file__), os.environ.get('JINA_DATA_FILE', None))
46 | with flow:
47 | flow.post(on='/index', inputs=input_generator(num_docs, data_path),
48 | show_progress=True)
49 |
50 |
51 | def query(top_k):
52 | flow = Flow().load_config('flows/flow.yml')
53 | with flow:
54 | text = input('Please type a sentence: ')
55 | doc = Document(content=text)
56 |
57 | result = flow.post(on='/search', inputs=DocumentArray([doc]),
58 | parameters={'top_k': top_k},
59 | line_format='text',
60 | return_results=True,
61 | )
62 | print_topk(result[0], text)
63 |
64 |
65 | @click.command()
66 | @click.option(
67 | '--task',
68 | '-t',
69 | type=click.Choice(['index', 'query'], case_sensitive=False),
70 | )
71 | @click.option('--num_docs', '-n', default=MAX_DOCS)
72 | @click.option('--top_k', '-k', default=5)
73 | @click.option('--dataset', '-d', type=click.Choice(['toy', 'full']), default='toy')
74 | def main(task, num_docs, top_k, dataset):
75 | config(dataset)
76 | if task == 'index':
77 | if os.path.exists(os.environ.get("JINA_WORKSPACE")):
78 | logger.error(f'\n +---------------------------------------------------------------------------------+ \
79 | \n | 🤖🤖🤖 | \
80 | \n | The directory {os.environ.get("JINA_WORKSPACE")} already exists. Please remove it before indexing again. | \
81 | \n | 🤖🤖🤖 | \
82 | \n +---------------------------------------------------------------------------------+')
83 | sys.exit(1)
84 | index(num_docs)
85 | elif task == 'query':
86 | query(top_k)
87 |
88 |
89 | if __name__ == '__main__':
90 | main()
91 |
--------------------------------------------------------------------------------
/wikipedia-sentences/data/toy-input.txt:
--------------------------------------------------------------------------------
1 | The ROMEO website, iOS app and Android app are commonly used by the male gay community to find friends, dates, love or get informed about LGBT+ topics.
2 | Once derided as corporate raiders, shareholder activists are now the recipients of admiration for sparking change in corporate boardrooms, leading to corporate boards developing best practices for responding to shareholder activism.
3 | Slc22a21 belongs to a protein family of solute carriers.
4 | Ajrara gharana or Ajrada gharana is one of the six main traditional schools in tabla drum.
5 | A few female specimens were found in a forest stream inside the shola forest.
6 | Ziggeo is the initiator and backer of BetaJS, an open-source framework.
7 | The three sports of aquatics were held at Aquatic Center in Sport Authority of Thailand Sport Complex, Bangkok, Thailand.
8 | Sugandha is the fourth generation of her family into singing and belongs to the Indore gharana.
9 | WYES is the only independently owned public television station in Louisiana as it is not part of Louisiana Public Broadcasting, which owns all of the PBS member stations in the state that are located outside of New Orleans, and maintains a programming agreement with and partial ownership of the city's independent public television station, WLAE-TV (channel 32).
10 | The reef divides the strait into the Apo East Pass and the Apo West Pass.
11 | His novel The Book of Evidence was shortlisted for the Booker Prize and won the Guinness Peat Aviation award in 1989.
12 | Andrea Kremer (born February 25, 1959 in Philadelphia, Pennsylvania) is a multi-Emmy Award Winning American television sports journalist.
13 | The book was the first published novel by O'Grady, with an initial print run of 6,000 hardback copies.
14 | After Alice performs several "miracle" cures in front of the tree, and claims to have seen the Virgin Mary there, it starts to be treated as a Lourdes-like shrine by Catholic pilgrims.
15 | Tovar is no longer involved with smuggling but acts as a consultant to Goldenvoice, which now operates the Coachella Valley Music and Arts Festival that has been compared to the Glastonbury Festival and is the most profitable music festival in the US.
16 | Tiwari worked as a producer with NDTV from 1996-2003.
17 | It is the home arena for SaPKo of the Mestis hockey league the second top league in Finland behind Liiga.
18 | As of the 2011 apportionment, the district includes the Middlesex County municipalities of East Brunswick Township, Edison Township, Helmetta Borough, Highland Park Borough, Metuchen Borough, South Plainfield Borough and South River Borough.
19 | Lembosiella is a genus of fungi in the Microthyriaceae family; according to the 2007 Outline of Ascomycota, the placement in this family is uncertain.
20 | Later, he resigned from his teaching profession in Jan 2013 and became a full time lyricist, dialog writer and part time researcher in Karky Research Foundation.
21 | It is used in Intel Core microarchitecture based DP-capable server processors, the Dual-Core Xeon is codenamed Dempsey, Woodcrest, and Wolfdale and the Quad-Core processors Clovertown, Harpertown.
22 | The 35th Annual TV Week Logie Awards was held on Friday 19 March 1993 at the Grand Hyatt in Melbourne, and broadcast on Network Ten.
23 | Daund Patas Road railway station is a small railway station in Pune district, Maharashtra.
24 | Evagjelia Veli (born 16 July 1991) is an Albanian weightlifter.
25 | It was published in two volumes that appeared a decade apart.
26 | He is now professor of medicine (biotechnology in public health) at the University of Bergen and chairs the Faculty Council, Faculty of Medicine, Norwegian University of Science and Technology.
27 | Erin McGathy (born December 5, 1985) is an American podcast host, artist, and comedian.
28 | The song is the second single from their debut mini album First Invasion and it was released as a digital single on August 4, 2010.
29 | Chandru who makes his debut in direction after assisting few Tamil films.
30 | Mitchum got the tune for the song from a Norwegian folk-dance (Gammel Reinlender) song his mother used to sing to him.
31 | The shell of the No 69 grenade was composed entirely of the hard plastic, Bakelite, which shattered without producing fragments like a metal bodied grenade.
32 | It was released on 24 January 2014.
33 | Stafford Loans are available both as subsidized and unsubsidized loans.
34 | On the World Wide Web, a query string is the part of a uniform resource locator (URL) containing data that does not fit conveniently into a hierarchical path structure.
35 | They have bar eyes, bare metasternum, bare metapisternum, the anterior anepisternum is usually pillose.
36 | In 1904, the mayoral term was changed to two years.
37 | It was earlier known as Central Mall but underwent renovations and some parts were re-organized in 2017 and was re-branded and re-launced on 26 February 2018.
38 | Jang Young-sik (born 1935) is a South Korean economist.
39 | It is a medium-sized damselfly with a short stout body, it is black with blue markings, and has long dark wings with pterostigma.
40 | The first desegregated hotel casino, it was popular with many of the black entertainers of the time, who would entertain at the other hotels and casinos and stay at the Moulin Rouge.
41 | In February 2009, it was revealed that the site was projected onto a wall at The Daily Telegraph to allow journalists there to view breaking news posted by users to Twitter.
42 | His most recent novel in this series, The Bangkok Asset, was published on 4 August 2015.
43 | He served as the 24th Governor of Nevada from 1979 to 1983.
44 | The soils which range from acid to alkaline and front wet to dry gives rise to a diverse woodland structure.
45 | Their land was taken back by the Spanish Crown; and then irretrievably lost however, when California became part of the United States.
46 | With annual billings of $220 million, Tombras is one of the top 25 largest independent national advertising agencies.
47 | The couple intended to retire to China and purchased a property in Canton; however the Communist victory in 1949 changed their plans and in 1950 the couple sold the vineyard and moved to Blockhouse Bay, Auckland.
48 | "Super Scooter Happy" was covered by Kyary Pamyu Pamyu on her 2013 album, Nanda Collection.
49 | Filling four CD-ROMs, Final Fantasy IX featured a cast containing a variety of major and minor characters.
50 | The album was produced by Billy Harvey, and featured contributions by Rafael Gayol and the Tosca String Quartet.
51 |
--------------------------------------------------------------------------------
/wikipedia-sentences/flows/flow.yml:
--------------------------------------------------------------------------------
1 | jtype: Flow # This file defines the flow (both index and query) for the wikipedia sentences example
2 | version: '1' # This is the yml file version
3 | with: # Additional arguments for the flow
4 | workspace: $JINA_WORKSPACE # Workspace folder path
5 | port_expose: $JINA_PORT # Network Port for the flow
6 | executors: # Now, define the executors that are run on this flow
7 | - name: transformer # This executor computes an embedding based on the input text documents
8 | uses: 'jinahub+docker://TransformerTorchEncoder/v0.1' # We use a Transformer Torch Encoder from the hub as a docker container
9 | - name: indexer # Now, index the text documents with the embeddings
10 | uses: 'jinahub://SimpleIndexer/old' # We use the SimpleIndexer for this purpose
--------------------------------------------------------------------------------
/wikipedia-sentences/get_data.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | DATASET="mikeortman/wikipedia-sentences"
3 | DATA_DIR="data"
4 | LINES=3000
5 |
6 |
7 |
8 | cd ${DATA_DIR}
9 | kaggle datasets download -d ${DATASET}
10 | unzip wikipedia-sentences.zip
11 | rm -f toy-data.txt
12 | rm -f wikipedia-sentences.zip
13 | mv wikisent2.txt input.txt
14 |
--------------------------------------------------------------------------------
/wikipedia-sentences/requirements.txt:
--------------------------------------------------------------------------------
1 | click==8.0.1
2 | jina[standard]==2.0.18
3 | git+git://github.com/jina-ai/jina-commons@v0.0.3
--------------------------------------------------------------------------------
/wikipedia-sentences/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jina-ai/examples/d8f903278597254bd96b1ed64fe8c9feefaa265c/wikipedia-sentences/tests/__init__.py
--------------------------------------------------------------------------------
/wikipedia-sentences/tests/conftest.py:
--------------------------------------------------------------------------------
1 | """Patch the birthday problem for random parts"""
2 |
3 | import pytest
4 |
5 |
6 | @pytest.fixture(scope='function', autouse=True)
7 | def patched_random_port(mocker):
8 | used_ports = set()
9 | from jina.helper import random_port
10 | from jina.excepts import NoAvailablePortError
11 |
12 | def _random_port():
13 |
14 | for i in range(10):
15 | _port = random_port()
16 |
17 | if _port is not None and _port not in used_ports:
18 | used_ports.add(_port)
19 | return _port
20 | raise NoAvailablePortError
21 |
22 | mocker.patch('jina.helper.random_port', new_callable=lambda: _random_port)
--------------------------------------------------------------------------------
/wikipedia-sentences/tests/requirements.txt:
--------------------------------------------------------------------------------
1 | pytest==6.2.4
2 | click==8.0.1
3 | jina[standard]==2.0.18
4 | git+git://github.com/jina-ai/jina-commons@v0.0.3
--------------------------------------------------------------------------------
/wikipedia-sentences/tests/test_wikipediasearch.py:
--------------------------------------------------------------------------------
1 | __copyright__ = "Copyright (c) 2021 Jina AI Limited. All rights reserved."
2 | __license__ = "Apache-2.0"
3 |
4 | import os
5 | import sys
6 | from click.testing import CliRunner
7 |
8 | sys.path.append('..')
9 | from app import main
10 |
11 |
12 | def config(tmpdir):
13 | os.environ['JINA_WORKSPACE'] = os.path.join(tmpdir, 'workspace')
14 |
15 |
16 | def test_wikipedia_sentences(tmpdir):
17 | config(tmpdir)
18 | runner = CliRunner()
19 | result = runner.invoke(main, ['-t', 'index'])
20 | assert "done in" in result.stdout
21 | assert result.stderr_bytes is None
22 | result = runner.invoke(main, ['-t', 'query'])
23 | print(result.stdout)
24 | assert result.stderr_bytes is None
25 |
--------------------------------------------------------------------------------
/wikipedia-sentences/tests/toy-input.txt:
--------------------------------------------------------------------------------
1 | ../data/toy-input.txt
--------------------------------------------------------------------------------
34 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
35 |
67 |
36 |
45 |
37 | Query
38 |
39 |
40 |
41 |
42 |
44 | {{ searchIndicator }}
43 |
46 |
66 |
47 | Breakdown
48 |
51 |
52 |
53 |
65 |
60 | {{searchQuery.substring(item.location[0], item.location[1])}}
63 |
64 |
68 |
99 |
69 |
98 |
70 | Top-{{topkDocs.length}} Results
71 |
72 |
73 |
97 |
75 |
96 |
77 |
95 |
78 |
94 |
79 |
93 | 80 | 82 | 87 |88 |
89 | {{index+1}} 90 | Relevance: {{ 1 - item.scores["cosine"].value.toFixed(3) }} 91 |
92 |