Retrieved {len(search_results):,.0f} documents in {search_time*1000:.2f} ms
', unsafe_allow_html=True) 28 | for result in search_results[:10]: 29 | result = json.loads(result.raw) 30 | doc = result["contents"] 31 | result_id = result["id"] 32 | try: 33 | st.write(doc[:1000], unsafe_allow_html=True) 34 | st.write(f'{result[250:]}
{{ cookiecutter.space_title }}
""") 43 | with gr.Row(): 44 | with gr.Column(scale=1): 45 | result_list = gr.Dataframe(type="array", visible=False, col_count=1) 46 | with gr.Column(scale=13): 47 | query = gr.Textbox(lines=1, max_lines=1, placeholder="Search…", label="") 48 | with gr.Column(scale=1): 49 | with gr.Row(scale=1): 50 | pass 51 | with gr.Row(scale=1): 52 | submit_btn = gr.Button("🔍", elem_id="b").style(full_width=False) 53 | with gr.Row(scale=1): 54 | pass 55 | 56 | with gr.Row(): 57 | with gr.Column(scale=1): 58 | pass 59 | with gr.Column(scale=13): 60 | c = gr.HTML(label="Results") 61 | with gr.Row(visible=False) as pagination: 62 | # left = gr.Button(value="◀", elem_id="b", visible=False).style(full_width=True) 63 | page_1 = gr.Button(value="1", elem_id="b").style(full_width=True) 64 | page_2 = gr.Button(value="2", elem_id="b").style(full_width=True) 65 | page_3 = gr.Button(value="3", elem_id="b").style(full_width=True) 66 | page_4 = gr.Button(value="4", elem_id="b").style(full_width=True) 67 | page_5 = gr.Button(value="5", elem_id="b").style(full_width=True) 68 | page_6 = gr.Button(value="6", elem_id="b").style(full_width=True) 69 | page_7 = gr.Button(value="7", elem_id="b").style(full_width=True) 70 | page_8 = gr.Button(value="8", elem_id="b").style(full_width=True) 71 | page_9 = gr.Button(value="9", elem_id="b").style(full_width=True) 72 | page_10 = gr.Button(value="10", elem_id="b").style(full_width=True) 73 | # right = gr.Button(value="▶", elem_id="b", visible=False).style(full_width=True) 74 | with gr.Column(scale=1): 75 | pass 76 | 77 | query.submit(fn=page_0, inputs=[query], outputs=[c, result_list, pagination]) 78 | submit_btn.click(page_0, inputs=[query], outputs=[c, result_list, pagination]) 79 | 80 | with gr.Box(visible=False): 81 | nums = [gr.Number(i, visible=False, precision=0) for i in range(NUM_PAGES)] 82 | 83 | page_1.click(fn=page_i, inputs=[nums[0], result_list], outputs=[c, result_list]) 84 | page_2.click(fn=page_i, inputs=[nums[1], result_list], outputs=[c, result_list]) 85 | page_3.click(fn=page_i, inputs=[nums[2], result_list], outputs=[c, result_list]) 86 | page_4.click(fn=page_i, inputs=[nums[3], result_list], outputs=[c, result_list]) 87 | page_5.click(fn=page_i, inputs=[nums[4], result_list], outputs=[c, result_list]) 88 | page_6.click(fn=page_i, inputs=[nums[5], result_list], outputs=[c, result_list]) 89 | page_7.click(fn=page_i, inputs=[nums[6], result_list], outputs=[c, result_list]) 90 | page_8.click(fn=page_i, inputs=[nums[7], result_list], outputs=[c, result_list]) 91 | page_9.click(fn=page_i, inputs=[nums[8], result_list], outputs=[c, result_list]) 92 | page_10.click(fn=page_i, inputs=[nums[9], result_list], outputs=[c, result_list]) 93 | 94 | demo.launch(enable_queue=True, debug=True) 95 | -------------------------------------------------------------------------------- /templates/gradio_roots_temp/{{ cookiecutter.local_app }}/app.py: -------------------------------------------------------------------------------- 1 | from typing import List, NewType, Optional, Union 2 | 3 | import gradio as gr 4 | 5 | from spacerini_utils.index import fetch_index_stats 6 | from spacerini_utils.search import _search, init_searcher, SearchResult 7 | 8 | HTML = NewType('HTML', str) 9 | 10 | searcher = init_searcher(sparse_index_path="sparse_index") 11 | 12 | def get_docid_html(docid: Union[int, str]) -> HTML: 13 | {% if cookiecutter.private -%} 14 | docid_html = ( 15 | f"🔒{{ cookiecutter.dataset_name }}/'+f'{docid}' 20 | ) 21 | {%- else -%} 22 | docid_html = ( 23 | f"🔒{{ cookiecutter.dataset_name }}/'+f'{docid}' 29 | ) 30 | {%- endif %} 31 | 32 | return docid_html 33 | 34 | 35 | def process_results(results: List[SearchResult], language: str, highlight_terms: Optional[List[str]] = None) -> HTML: 36 | if len(results) == 0: 37 | return """38 | No results retrieved.
56 | """ 57 | ) 58 | docid_html = get_docid_html(result["docid"]) 59 | results_html += """{} 60 |
Document ID: {}
61 |Score: {}
62 |Language: {}
63 |{}
64 |{{ cookiecutter.space_description}}
80 |Dataset Statistics: Total Number of Documents = {stats["documents"]}, Number of Terms = {stats["total_terms"]}
""" 81 | 82 | demo = gr.Blocks( 83 | css=".underline-on-hover:hover { text-decoration: underline; } .flagging { font-size:12px; color:Silver; }" 84 | ) 85 | 86 | with demo: 87 | with gr.Row(): 88 | gr.Markdown(value=description) 89 | with gr.Row(): 90 | query = gr.Textbox(lines=1, max_lines=1, placeholder="Type your query here...", label="Query") 91 | with gr.Row(): 92 | lang = gr.Dropdown( 93 | choices=[ 94 | "en", 95 | "detect_language", 96 | "all", 97 | ], 98 | value="en", 99 | label="Language", 100 | ) 101 | with gr.Row(): 102 | k = gr.Slider(1, 100, value=10, step=1, label="Max Results") 103 | with gr.Row(): 104 | submit_btn = gr.Button("Submit") 105 | with gr.Row(): 106 | results = gr.HTML(label="Results") 107 | 108 | 109 | def submit(query: str, lang: str, k: int): 110 | query = query.strip() 111 | if query is None or query == "": 112 | return "", "" 113 | return { 114 | results: search(query, lang, k), 115 | } 116 | 117 | query.submit(fn=submit, inputs=[query, lang, k], outputs=[results]) 118 | submit_btn.click(submit, inputs=[query, lang, k], outputs=[results]) 119 | 120 | demo.launch(enable_queue=True, debug=True) 121 | -------------------------------------------------------------------------------- /src/spacerini/data/load.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datasets import Dataset, IterableDataset 3 | from datasets import load_dataset 4 | import pandas as pd 5 | import ir_datasets 6 | from typing import Generator, Dict, List 7 | 8 | def ir_dataset_dict_generator(dataset_name: str) -> Generator[Dict,None,None]: 9 | """ 10 | Generator for streaming datasets from ir_datasets 11 | https://github.com/allenai/ir_datasets 12 | Parameters 13 | ---------- 14 | dataset_name : str 15 | Name of dataset to load 16 | """ 17 | dataset = ir_datasets.load(dataset_name) 18 | for doc in dataset.docs_iter(): 19 | yield { 20 | "contents": doc.text, 21 | "docid": doc.doc_id 22 | } 23 | 24 | def load_ir_dataset(dataset_name: str) -> Dataset: 25 | """ 26 | Load dataset from ir_datasets 27 | 28 | Parameters 29 | ---------- 30 | dataset_name : str 31 | Name of dataset to load 32 | 33 | Returns 34 | ------- 35 | Dataset 36 | """ 37 | dataset = ir_datasets.load(dataset_name) 38 | return Dataset.from_pandas(pd.DataFrame(dataset.docs_iter())) 39 | 40 | def load_ir_dataset_low_memory(dataset_name: str, num_proc: int) -> IterableDataset: 41 | """ 42 | Load dataset from ir_datasets by streaming into a Dataset object. This is slower than first loading the data into a pandas DataFrame but does not require loading the entire ir_dataset into memory. This variant also supports multiprocessing if the data is sharded, and only consumes the generator once then caches it so that future calls are instantaneous. 43 | 44 | Parameters 45 | ---------- 46 | dataset_name : str 47 | Name of dataset to load 48 | num_proc: int 49 | Number of processes to use 50 | 51 | Returns 52 | ------- 53 | Dataset 54 | """ 55 | return Dataset.from_generator(ir_dataset_dict_generator, gen_kwargs={"dataset_name": dataset_name}, num_proc=num_proc) 56 | 57 | def load_ir_dataset_streaming(dataset_name: str) -> IterableDataset: 58 | """ 59 | Load dataset from ir_datasets 60 | 61 | Parameters 62 | ---------- 63 | dataset_name : str 64 | Name of dataset to load 65 | 66 | Returns 67 | ------- 68 | IterableDataset 69 | """ 70 | return IterableDataset.from_generator(ir_dataset_dict_generator, gen_kwargs={"dataset_name": dataset_name}) 71 | 72 | def load_from_pandas(df: pd.DataFrame) -> Dataset: 73 | """ 74 | Load dataset from pandas DataFrame 75 | 76 | Parameters 77 | ---------- 78 | df : pd.DataFrame 79 | DataFrame to load 80 | 81 | Returns 82 | ------- 83 | Dataset 84 | """ 85 | return Dataset.from_pandas(df) 86 | 87 | def load_from_hub(dataset_name_or_path: str, split: str, config_name: str=None, streaming: bool = True) -> Dataset: 88 | """ 89 | Load dataset from HuggingFace Hub 90 | 91 | Parameters 92 | ---------- 93 | dataset_name_or_path : str 94 | Name of dataset to load 95 | split : str 96 | Split of dataset to load 97 | streaming : bool 98 | Whether to load dataset in streaming mode 99 | 100 | Returns 101 | ------- 102 | Dataset 103 | """ 104 | 105 | return load_dataset(dataset_name_or_path, split=split, streaming=streaming, name=config_name) 106 | 107 | def load_from_local(dataset_name_or_path: str or List[str], split: str, streaming: bool = True) -> Dataset: 108 | """ 109 | Load dataset from local text file. Supports JSON, JSONL, CSV, and TSV. 110 | 111 | Parameters 112 | ---------- 113 | dataset_name_or_path : str 114 | Path to a .json, .jsonl, .csv, or .tsv file 115 | streaming : bool 116 | Whether to load dataset in streaming mode 117 | 118 | Returns 119 | ------- 120 | Dataset 121 | """ 122 | if isinstance(dataset_name_or_path, str): 123 | assert os.path.exists(dataset_name_or_path), f"File {dataset_name_or_path} does not exist" 124 | if dataset_name_or_path.endswith(".jsonl") or dataset_name_or_path.endswith(".json"): 125 | return load_dataset("json", data_files=dataset_name_or_path, split=split, streaming=streaming) 126 | elif dataset_name_or_path.endswith(".csv"): 127 | return load_dataset("csv", data_files=dataset_name_or_path, split=split, streaming=streaming, sep=",") 128 | elif dataset_name_or_path.endswith(".tsv"): 129 | return load_dataset("csv", data_files=dataset_name_or_path, split=split, streaming=streaming, sep="\t") 130 | else: 131 | raise ValueError("Unsupported file type") 132 | 133 | def load_from_sqlite_table(uri_or_con: str, table_or_query: str) -> Dataset: 134 | """ 135 | Load dataset from a sqlite database table 136 | 137 | Parameters 138 | ---------- 139 | uri_or_con : str 140 | URI to a SQLITE database or connection object 141 | table_or_query : str 142 | database table or query that returns a table 143 | 144 | Returns 145 | ------- 146 | Dataset 147 | """ 148 | return Dataset.from_sql(con=uri_or_con, sql=table_or_query) 149 | -------------------------------------------------------------------------------- /src/spacerini/search/utils.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Dict 3 | from typing import List 4 | from typing import Literal 5 | from typing import Protocol 6 | from typing import TypedDict 7 | from typing import Union 8 | 9 | from datasets import Dataset 10 | from pyserini.analysis import get_lucene_analyzer 11 | from pyserini.search import DenseSearchResult 12 | from pyserini.search import JLuceneSearcherResult 13 | from pyserini.search.faiss.__main__ import init_query_encoder 14 | from pyserini.search.faiss import FaissSearcher 15 | from pyserini.search.hybrid import HybridSearcher 16 | from pyserini.search.lucene import LuceneSearcher 17 | 18 | Encoder = Literal["dkrr", "dpr", "tct_colbert", "ance", "sentence", "contriever", "auto"] 19 | 20 | 21 | class AnalyzerArgs(TypedDict): 22 | language: str 23 | stemming: bool 24 | stemmer: str 25 | stopwords: bool 26 | huggingFaceTokenizer: str 27 | 28 | 29 | class Searcher(Protocol): 30 | def search(self, query: str, **kwargs) -> List[Union[DenseSearchResult, JLuceneSearcherResult]]: 31 | ... 32 | 33 | 34 | def init_searcher( 35 | sparse_index_path: str = None, 36 | bm25_k1: float = None, 37 | bm25_b: float = None, 38 | analyzer_args: AnalyzerArgs = None, 39 | dense_index_path: str = None, 40 | encoder_name_or_path: str = None, 41 | encoder_class: Encoder = None, 42 | tokenizer_name: str = None, 43 | device: str = None, 44 | prefix: str = None 45 | ) -> Union[FaissSearcher, HybridSearcher, LuceneSearcher]: 46 | """ 47 | Initialize and return an approapriate searcher 48 | 49 | Parameters 50 | ---------- 51 | sparse_index_path: str 52 | Path to sparse index 53 | dense_index_path: str 54 | Path to dense index 55 | encoder_name_or_path: str 56 | Path to query encoder checkpoint or encoder name 57 | encoder_class: str 58 | Query encoder class to use. If None, infer from `encoder` 59 | tokenizer_name: str 60 | Tokenizer name or path 61 | device: str 62 | Device to load Query encoder on. 63 | prefix: str 64 | Query prefix if exists 65 | 66 | Returns 67 | ------- 68 | Searcher: 69 | A sparse, dense or hybrid searcher 70 | """ 71 | if sparse_index_path: 72 | ssearcher = LuceneSearcher(sparse_index_path) 73 | if analyzer_args: 74 | analyzer = get_lucene_analyzer(**analyzer_args) 75 | ssearcher.set_analyzer(analyzer) 76 | if bm25_k1 and bm25_b: 77 | ssearcher.set_bm25(bm25_k1, bm25_b) 78 | 79 | if dense_index_path: 80 | encoder = init_query_encoder( 81 | encoder=encoder_name_or_path, 82 | encoder_class=encoder_class, 83 | tokenizer_name=tokenizer_name, 84 | topics_name=None, 85 | encoded_queries=None, 86 | device=device, 87 | prefix=prefix 88 | ) 89 | 90 | dsearcher = FaissSearcher(dense_index_path, encoder) 91 | 92 | if sparse_index_path: 93 | hsearcher = HybridSearcher(dense_searcher=dsearcher, sparse_searcher=ssearcher) 94 | return hsearcher 95 | else: 96 | return dsearcher 97 | 98 | return ssearcher 99 | 100 | 101 | def result_indices( 102 | query: str, 103 | num_results: int, 104 | searcher: Searcher, 105 | index_path: str, 106 | analyzer=None 107 | ) -> list: 108 | """ 109 | Get the indices of the results of a query. 110 | Parameters 111 | ---------- 112 | query : str 113 | The query. 114 | num_results : int 115 | The number of results to return. 116 | index_path : str 117 | The path to the index. 118 | analyzer : str (default=None) 119 | The analyzer to use. 120 | 121 | Returns 122 | ------- 123 | list 124 | The indices of the returned documents. 125 | """ 126 | # searcher.search() 127 | # searcher = LuceneSearcher(index_path) 128 | # if analyzer is not None: 129 | # searcher.set_analyzer(analyzer) 130 | hits = searcher.search(query, k=num_results) 131 | ix = [int(hit.docid) for hit in hits] 132 | return ix 133 | 134 | 135 | def result_page( 136 | hf_dataset: Dataset, 137 | result_indices: List[int], 138 | page: int = 0, 139 | results_per_page: int=10 140 | ) -> Dataset: 141 | """ 142 | Returns a the ith results page as a datasets.Dataset object. Nothing is loaded into memory. Call `to_pandas()` on the returned Dataset to materialize the table. 143 | ---------- 144 | hf_dataset : datasets.Dataset 145 | a Hugging Face datasets dataset. 146 | result_indices : list of int 147 | The indices of the results. 148 | page: int (default=0) 149 | The result page to return. Returns the first page by default. 150 | results_per_page : int (default=10) 151 | The number of results per page. 152 | 153 | Returns 154 | ------- 155 | datasets.Dataset 156 | A results page. 157 | """ 158 | results = hf_dataset.select(result_indices) 159 | num_result_pages = int(len(results)/results_per_page) + 1 160 | return results.shard(num_result_pages, page, contiguous=True) 161 | -------------------------------------------------------------------------------- /src/spacerini/index/index.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | from typing import List 3 | from typing import Literal 4 | 5 | import shutil 6 | from pyserini.index.lucene import LuceneIndexer, IndexReader 7 | from typing import Any, Dict, List 8 | from pyserini.pyclass import autoclass 9 | from tqdm import tqdm 10 | import json 11 | import os 12 | 13 | from spacerini.data import load_from_hub, load_from_local 14 | 15 | 16 | def parse_args( 17 | index_path: str, 18 | shards_path: str = "", 19 | fields: List[str] = None, 20 | language: str = None, 21 | pretokenized: bool = False, 22 | analyzeWithHuggingFaceTokenizer: str = None, 23 | storePositions: bool = True, 24 | storeDocvectors: bool = False, 25 | storeContents: bool = False, 26 | storeRaw: bool = False, 27 | keepStopwords: bool = False, 28 | stopwords: str = None, 29 | stemmer: Literal["porter", "krovetz"] = None, 30 | optimize: bool = False, 31 | verbose: bool = False, 32 | quiet: bool = False, 33 | memory_buffer: str ="4096", 34 | n_threads: int = 5, 35 | for_otf_indexing: bool = False, 36 | **kwargs 37 | ) -> List[str]: 38 | """ 39 | Parse arguments into list for `SimpleIndexer` class in Anserini. 40 | 41 | Parameters 42 | ---------- 43 | for_otf_indexing : bool 44 | If True, `-input` & `-collection` args are safely ignored. 45 | Used when performing on-the-fly indexing with HF Datasets. 46 | 47 | See [docs](docs/arguments.md) for remaining argument definitions 48 | 49 | Returns 50 | ------- 51 | List of arguments to initialize the `LuceneIndexer` 52 | """ 53 | params = locals() 54 | args = [] 55 | args.extend([ 56 | "-input", shards_path, 57 | "-collection", "JsonCollection", 58 | "-threads", f"{n_threads}" if n_threads!=-1 else f"{os.cpu_count()}", 59 | "-generator", "DefaultLuceneDocumentGenerator", 60 | "-index", index_path, 61 | "-memorybuffer", memory_buffer, 62 | ]) 63 | variables = [ 64 | "analyzeWithHuggingFaceTokenizer", "language", "stemmer" 65 | ] 66 | additional_args = [[f"-{var}", params[var]] for var in variables if params[var]] 67 | args.extend(chain.from_iterable(additional_args)) 68 | 69 | if fields: 70 | args.extend(["-fields", " ".join(fields)]) 71 | 72 | flags = [ 73 | "pretokenized", "storePositions", "storeDocvectors", 74 | "storeContents", "storeRaw", "optimize", "verbose", "quiet" 75 | ] 76 | args.extend([f"-{flag}" for flag in flags if params[flag]]) 77 | 78 | if for_otf_indexing: 79 | args = args[4:] 80 | return args 81 | 82 | 83 | def index_json_shards( 84 | shards_path: str, 85 | index_path: str, 86 | keep_shards: bool = True, 87 | fields: List[str] = None, 88 | language: str = "en", 89 | pretokenized: bool = False, 90 | analyzeWithHuggingFaceTokenizer: str = None, 91 | storePositions: bool = True, 92 | storeDocvectors: bool = False, 93 | storeContents: bool = False, 94 | storeRaw: bool = False, 95 | keepStopwords: bool = False, 96 | stopwords: str = None, 97 | stemmer: Literal["porter", "krovetz"] = "porter", 98 | optimize: bool = True, 99 | verbose: bool = False, 100 | quiet: bool = False, 101 | memory_buffer: str = "4096", 102 | n_threads: bool = 5, 103 | ): 104 | """Index dataset from a directory containing files 105 | 106 | Parameters 107 | ---------- 108 | shards_path : str 109 | Path to dataset to index 110 | index_path : str 111 | Directory to store index 112 | keep_shards : bool 113 | If False, remove dataset after indexing is complete 114 | 115 | See [docs](../../docs/arguments.md) for remaining argument definitions 116 | 117 | Returns 118 | ------- 119 | None 120 | """ 121 | args = parse_args(**locals()) 122 | JIndexCollection = autoclass('io.anserini.index.IndexCollection') 123 | JIndexCollection.main(args) 124 | if not keep_shards: 125 | shutil.rmtree(shards_path) 126 | 127 | return None 128 | 129 | 130 | def index_streaming_dataset( 131 | index_path: str, 132 | dataset_name_or_path: str, 133 | split: str, 134 | column_to_index: List[str], 135 | doc_id_column: str = None, 136 | ds_config_name: str = None, # For HF Dataset 137 | num_rows: int = -1, 138 | disable_tqdm: bool = False, 139 | language: str = "en", 140 | pretokenized: bool = False, 141 | analyzeWithHuggingFaceTokenizer: str = None, 142 | storePositions: bool = True, 143 | storeDocvectors: bool = False, 144 | storeContents: bool = False, 145 | storeRaw: bool = False, 146 | keepStopwords: bool = False, 147 | stopwords: str = None, 148 | stemmer: Literal["porter", "krovetz"] = "porter", 149 | optimize: bool = True, 150 | verbose: bool = False, 151 | quiet: bool = True, 152 | memory_buffer: str = "4096", 153 | n_threads: bool = 5, 154 | ): 155 | """Stream dataset from HuggingFace Hub & index 156 | 157 | Parameters 158 | ---------- 159 | dataset_name_or_path : str 160 | Name of HuggingFace dataset to stream 161 | split : str 162 | Split of dataset to index 163 | column_to_index : List[str] 164 | Column of dataset to index 165 | doc_id_column : str 166 | Column of dataset to use as document ID 167 | ds_config_name: str 168 | Dataset configuration to stream. Usually a language name or code 169 | num_rows : int 170 | Number of rows in dataset 171 | 172 | See [docs](../../docs/arguments.md) for remaining argument definitions 173 | 174 | Returns 175 | ------- 176 | None 177 | """ 178 | 179 | args = parse_args(**locals(), for_otf_indexing=True) 180 | if os.path.exists(dataset_name_or_path): 181 | ds = load_from_local(dataset_name_or_path, split=split, streaming=True) 182 | else: 183 | ds = load_from_hub(dataset_name_or_path, split=split,config_name=ds_config_name, streaming=True) 184 | 185 | indexer = LuceneIndexer(args=args) 186 | 187 | for i, row in tqdm(enumerate(ds), total=num_rows, disable=disable_tqdm): 188 | contents = " ".join([row[column] for column in column_to_index]) 189 | indexer.add_doc_raw(json.dumps({"id": i if not doc_id_column else row[doc_id_column] , "contents": contents})) 190 | 191 | indexer.close() 192 | 193 | return None 194 | 195 | 196 | def fetch_index_stats(index_path: str) -> Dict[str, Any]: 197 | """ 198 | Fetch index statistics 199 | index_path : str 200 | Path to index directory 201 | Returns 202 | ------- 203 | Dictionary of index statistics 204 | Dictionary Keys ==> total_terms, documents, unique_terms 205 | """ 206 | assert os.path.exists(index_path), f"Index path {index_path} does not exist" 207 | index_reader = IndexReader(index_path) 208 | return index_reader.stats() 209 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /src/spacerini/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | from pathlib import Path 5 | from shutil import copytree 6 | 7 | from spacerini.frontend import create_app, create_space_from_local 8 | from spacerini.index import index_streaming_dataset 9 | from spacerini.index.encode import encode_json_dataset 10 | from spacerini.prebuilt import EXAMPLES 11 | 12 | 13 | def update_args_from_json(_args: argparse.Namespace, file: str) -> argparse.Namespace: 14 | config = json.load(open(file, "r")) 15 | config = {k.replace("-", "_"): v for k,v in config.items()} 16 | 17 | args_dict = vars(_args) 18 | args_dict.update(config) 19 | return _args 20 | 21 | 22 | # TODO: @theyorubayesian: Switch to HFArgumentParser with post init 23 | # How to use multiple argparse commands with HFArgumentParser 24 | def get_args() -> argparse.Namespace: 25 | parser = argparse.ArgumentParser( 26 | prog="Spacerini", 27 | description="A modular framework for seamless building and deployment of interactive search applications.", 28 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 29 | epilog="Written by: Akintunde 'theyorubayesian' Oladipo