1# SPDX-FileCopyrightText: 2024-present Yuxuan Wang <wangy49@seas.upenn.edu> 64 | 2# 65 | 3# SPDX-License-Identifier: Apache-2.0 66 | 4from collections.abc import Iterable 67 | 5from functools import wraps 68 | 6from typing import Any, Callable, Final, Optional 69 | 7 70 | 8import pandas as pd 71 | 9from haystack.dataclasses import Document 72 | 10from haystack.errors import FilterError 73 | 11 74 | 12 75 | 13def apply_filters_to_document( 76 | 14 filters: Optional[dict[str, Any]], document: Document 77 | 15) -> bool: 78 | 16 """ 79 | 17 Apply filters to a document. 80 | 18 81 | 19 :param filters: The filters to apply to the document. 82 | 20 :type filters: dict[str, Any] 83 | 21 :param document: The document to apply the filters to. 84 | 22 :type document: Document 85 | 23 86 | 24 :return: True if the document passes the filters. 87 | 25 :rtype: bool 88 | 26 """ 89 | 27 if filters is None or not filters: 90 | 28 return True 91 | 29 return _run_comparison_condition(filters, document) 92 | 30 93 | 31 94 | 32def _get_document_field(document: Document, field: str) -> Optional[Any]: 95 | 33 """ 96 | 34 Get the value of a field in a document. 97 | 35 98 | 36 If the field is not found within the document then, instead of 99 | 37 raising an error, `None` is returned. Note that here we do not 100 | 38 implicitly add 'meta' prefix for fields that are not a direct 101 | 39 attribute of the document, not supporting legacy behavior anymore. 102 | 40 103 | 41 :param document: The document to get the field value from. 104 | 42 :type document: Document 105 | 43 :param field: The field to get the value of. 106 | 44 :type field: str 107 | 45 108 | 46 :return: The value of the field in the document. 109 | 47 :rtype: Optional[Any] 110 | 48 """ 111 | 49 if "." not in field: 112 | 50 return getattr(document, field) 113 | 51 114 | 52 attr = document.meta 115 | 53 for f in field.split(".")[1:]: 116 | 54 attr = attr.get(f) 117 | 55 if attr is None: 118 | 56 return None 119 | 57 return attr 120 | 58 121 | 59 122 | 60def _run_logical_condition(condition: dict[str, Any], document: Document) -> bool: 123 | 61 if "operator" not in condition: 124 | 62 msg = "Logical condition must have an 'operator' key." 125 | 63 raise FilterError(msg) 126 | 64 if "conditions" not in condition: 127 | 65 msg = "Logical condition must have a 'conditions' key." 128 | 66 raise FilterError(msg) 129 | 67 130 | 68 conditions = condition["conditions"] 131 | 69 reducer = LOGICAL_OPERATORS[condition["operator"]] 132 | 70 133 | 71 return reducer(document, conditions) 134 | 72 135 | 73 136 | 74def _run_comparison_condition(condition: dict[str, Any], document: Document) -> bool: 137 | 75 if "field" not in condition: 138 | 76 return _run_logical_condition(condition, document) 139 | 77 140 | 78 if "operator" not in condition: 141 | 79 msg = "Comparison condition must have an 'operator' key." 142 | 80 raise FilterError(msg) 143 | 81 if "value" not in condition: 144 | 82 msg = "Comparison condition must have a 'value' key." 145 | 83 raise FilterError(msg) 146 | 84 147 | 85 field: str = condition["field"] 148 | 86 value: Any = condition["value"] 149 | 87 comparator = COMPARISON_OPERATORS[condition["operator"]] 150 | 88 151 | 89 return comparator(_get_document_field(document, field), value) 152 | 90 153 | 91 154 | 92def _and(document: Document, conditions: list[dict[str, Any]]) -> bool: 155 | 93 """ 156 | 94 Return True if all conditions are met. 157 | 95 158 | 96 :param document: The document to check the conditions against. 159 | 97 :type document: Document 160 | 98 :param conditions: The conditions to check against the document. 161 | 99 :type conditions: list[dict[str, Any]] 162 | 100 163 | 101 :return: True if not all conditions are met. 164 | 102 :rtype: bool 165 | 103 """ 166 | 104 return all( 167 | 105 _run_comparison_condition(condition, document) for condition in conditions 168 | 106 ) 169 | 107 170 | 108 171 | 109def _or(document: Document, conditions: list[dict[str, Any]]) -> bool: 172 | 110 """ 173 | 111 Return True if any condition is met. 174 | 112 175 | 113 :param document: The document to check the conditions against. 176 | 114 :type document: Document 177 | 115 :param conditions: The conditions to check against the document. 178 | 116 :type conditions: list[dict[str, Any]] 179 | 117 180 | 118 :return: True if not all conditions are met. 181 | 119 :rtype: bool 182 | 120 """ 183 | 121 return any(_run_comparison_condition(cond, document) for cond in conditions) 184 | 122 185 | 123 186 | 124def _not(document: Document, conditions: list[dict[str, Any]]) -> bool: 187 | 125 """ 188 | 126 Return True if not all conditions are met. 189 | 127 190 | 128 The 'NOT' operator is under-specified when supplied with a 191 | 129 set of conditions instead of a single condition. Because we 192 | 130 can have the semantics of 'at least one False' versus 193 | 131 'all False'. Here we choose to comply with the official 194 | 132 implementation of Haystack (the 'at least one False' semantics). 195 | 133 196 | 134 :param document: The document to check the conditions against. 197 | 135 :type document: Document 198 | 136 :param conditions: The conditions to check against the document. 199 | 137 :type conditions: list[dict[str, Any]] 200 | 138 201 | 139 :return: True if not all conditions are met. 202 | 140 :rtype: bool 203 | 141 """ 204 | 142 return not _and(document, conditions) 205 | 143 206 | 144 207 | 145def _check_comparator_inputs( 208 | 146 comparator: Callable[[Any, Any], bool] 209 | 147) -> Callable[[Any, Any], bool]: 210 | 148 """ 211 | 149 A decorator to check and preprocess input attribute values. 212 | 150 213 | 151 ALL COMPARISON OPERATORS SHOULD BE WRAPPED WITH THIS DECORATOR. 214 | 152 because a `False` may be returned by both input validation and 215 | 153 the actual comparison. This decorator ensures that the comparison 216 | 154 function is only called if the input values are valid. 217 | 155 218 | 156 :param comparator: The comparator function to wrap. 219 | 157 :type comparator: Callable[[Any, Any], bool] 220 | 158 221 | 159 :return: The wrapped comparator function. 222 | 160 :rtype: Callable[[Any, Any], bool] 223 | 161 """ 224 | 162 225 | 163 @wraps(comparator) 226 | 164 def _wrapper(dv: Any, fv: Any) -> bool: 227 | 165 228 | 166 # I think allowing comparison between DataFrames would 229 | 167 # be a really bad idea because it would create unexpected 230 | 168 # behavior, but I am open to discussion on this. 231 | 169 if isinstance(dv, pd.DataFrame) or isinstance(fv, pd.DataFrame): 232 | 170 msg = ( 233 | 171 "Cannot compare DataFrames. Please convert them to " 234 | 172 "simpler data structures before comparing." 235 | 173 ) 236 | 174 raise FilterError(msg) 237 | 175 238 | 176 # I think comparison between missing values is ambiguous, 239 | 177 # but again, I am open to discussion on this. Here I choose 240 | 178 # to return False if either value is None because from a 241 | 179 # logical perspective, we really cannot say anything about 242 | 180 # the comparison between a missing value and a non-missing. 243 | 181 if dv is None or fv is None: 244 | 182 return False 245 | 183 246 | 184 try: 247 | 185 return comparator(dv, fv) 248 | 186 except TypeError as exc: 249 | 187 msg = ( 250 | 188 f"Cannot compare document value of {type(dv)} type " 251 | 189 f"with filter value of {type(fv)} type." 252 | 190 ) 253 | 191 raise FilterError(msg) from exc 254 | 192 255 | 193 return _wrapper 256 | 194 257 | 195 258 | 196@_check_comparator_inputs 259 | 197def _eq(dv: Any, fv: Any) -> bool: 260 | 198 """ 261 | 199 Conservative implementation of equal comparison. 262 | 200 263 | 201 There are two major differences between this implementation 264 | 202 and the default Haystack filter implementation: 265 | 203 - If both values are None, we return False, instead of True. 266 | 204 - If any value is a DataFrame, we raise an error, instead 267 | 205 of converting them to JSON. 268 | 206 """ 269 | 207 return dv == fv 270 | 208 271 | 209 272 | 210@_check_comparator_inputs 273 | 211def _ne(dv: Any, fv: Any) -> bool: 274 | 212 return not _eq(dv, fv) 275 | 213 276 | 214 277 | 215@_check_comparator_inputs 278 | 216def _gt(dv: Any, fv: Any) -> bool: 279 | 217 """ 280 | 218 A more liberal implementation with less surprises. 281 | 219 282 | 220 Simply compare the two values with default Python comparison. 283 | 221 We do not perform any conversion here to have the behavior 284 | 222 more predictable. If we want to compare the dates, we should 285 | 223 just convert the document value and filter value explicitly 286 | 224 to dates before comparing them. 287 | 225 """ 288 | 226 return dv > fv 289 | 227 290 | 228 291 | 229@_check_comparator_inputs 292 | 230def _lt(dv: Any, fv: Any) -> bool: 293 | 231 return dv < fv 294 | 232 295 | 233 296 | 234@_check_comparator_inputs 297 | 235def _gte(dv: Any, fv: Any) -> bool: 298 | 236 return _gt(dv, fv) or _eq(dv, fv) 299 | 237 300 | 238 301 | 239@_check_comparator_inputs 302 | 240def _lte(dv: Any, fv: Any) -> bool: 303 | 241 return _lt(dv, fv) or _eq(dv, fv) 304 | 242 305 | 243 306 | 244@_check_comparator_inputs 307 | 245def _in(dv: Any, fv: Any) -> bool: 308 | 246 """ 309 | 247 Allowing iterable filter values not just lists. 310 | 248 311 | 249 This implementation permits a larger set of filter values 312 | 250 such as tuples, sets, and other iterable objects. 313 | 251 """ 314 | 252 if not isinstance(fv, Iterable): 315 | 253 msg = "Filter value must be an iterable for 'in' comparison." 316 | 254 raise FilterError(msg) 317 | 255 318 | 256 return any(_eq(dv, v) for v in fv) 319 | 257 320 | 258 321 | 259@_check_comparator_inputs 322 | 260def _nin(dv: Any, fv: Any) -> bool: 323 | 261 return not _in(dv, fv) 324 | 262 325 | 263 326 | 264LOGICAL_OPERATORS: Final = {"NOT": _not, "AND": _and, "OR": _or} 327 | 265 328 | 266COMPARISON_OPERATORS: Final = { 329 | 267 "==": _eq, 330 | 268 "!=": _ne, 331 | 269 ">": _gt, 332 | 270 "<": _lt, 333 | 271 ">=": _gte, 334 | 272 "<=": _lte, 335 | 273 "in": _in, 336 | 274 "not in": _nin, 337 | 275} 338 |
14def apply_filters_to_document( 354 | 15 filters: Optional[dict[str, Any]], document: Document 355 | 16) -> bool: 356 | 17 """ 357 | 18 Apply filters to a document. 358 | 19 359 | 20 :param filters: The filters to apply to the document. 360 | 21 :type filters: dict[str, Any] 361 | 22 :param document: The document to apply the filters to. 362 | 23 :type document: Document 363 | 24 364 | 25 :return: True if the document passes the filters. 365 | 26 :rtype: bool 366 | 27 """ 367 | 28 if filters is None or not filters: 368 | 29 return True 369 | 30 return _run_comparison_condition(filters, document) 370 |
Apply filters to a document.
374 | 375 |385 |387 |True if the document passes the filters.
386 |