2685 lines
112 KiB
Python
2685 lines
112 KiB
Python
"""
|
|
Create SQL statements for QuerySets.
|
|
|
|
The code in here encapsulates all of the SQL construction so that QuerySets
|
|
themselves do not have to (and could be backed by things other than SQL
|
|
databases). The abstraction barrier only works one way: this module has to know
|
|
all about the internals of models in order to get the information it needs.
|
|
"""
|
|
import copy
|
|
import difflib
|
|
import functools
|
|
import sys
|
|
from collections import Counter, namedtuple
|
|
from collections.abc import Iterator, Mapping
|
|
from itertools import chain, count, product
|
|
from string import ascii_uppercase
|
|
|
|
from django.core.exceptions import FieldDoesNotExist, FieldError
|
|
from django.db import DEFAULT_DB_ALIAS, NotSupportedError, connections
|
|
from django.db.models.aggregates import Count
|
|
from django.db.models.constants import LOOKUP_SEP
|
|
from django.db.models.expressions import (
|
|
BaseExpression,
|
|
Col,
|
|
Exists,
|
|
F,
|
|
OuterRef,
|
|
Ref,
|
|
ResolvedOuterRef,
|
|
Value,
|
|
)
|
|
from django.db.models.fields import Field
|
|
from django.db.models.fields.related_lookups import MultiColSource
|
|
from django.db.models.lookups import Lookup
|
|
from django.db.models.query_utils import (
|
|
Q,
|
|
check_rel_lookup_compatibility,
|
|
refs_expression,
|
|
)
|
|
from django.db.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE
|
|
from django.db.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin
|
|
from django.db.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode
|
|
from django.utils.functional import cached_property
|
|
from django.utils.regex_helper import _lazy_re_compile
|
|
from django.utils.tree import Node
|
|
|
|
__all__ = ["Query", "RawQuery"]
|
|
|
|
# Quotation marks ('"`[]), whitespace characters, semicolons, or inline
|
|
# SQL comments are forbidden in column aliases.
|
|
FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(r"['`\"\]\[;\s]|--|/\*|\*/")
|
|
|
|
# Inspired from
|
|
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
|
|
EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w\-]+")
|
|
|
|
|
|
def get_field_names_from_opts(opts):
|
|
if opts is None:
|
|
return set()
|
|
return set(
|
|
chain.from_iterable(
|
|
(f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields()
|
|
)
|
|
)
|
|
|
|
|
|
def get_children_from_q(q):
|
|
for child in q.children:
|
|
if isinstance(child, Node):
|
|
yield from get_children_from_q(child)
|
|
else:
|
|
yield child
|
|
|
|
|
|
JoinInfo = namedtuple(
|
|
"JoinInfo",
|
|
("final_field", "targets", "opts", "joins", "path", "transform_function"),
|
|
)
|
|
|
|
|
|
class RawQuery:
|
|
"""A single raw SQL query."""
|
|
|
|
def __init__(self, sql, using, params=()):
|
|
self.params = params
|
|
self.sql = sql
|
|
self.using = using
|
|
self.cursor = None
|
|
|
|
# Mirror some properties of a normal query so that
|
|
# the compiler can be used to process results.
|
|
self.low_mark, self.high_mark = 0, None # Used for offset/limit
|
|
self.extra_select = {}
|
|
self.annotation_select = {}
|
|
|
|
def chain(self, using):
|
|
return self.clone(using)
|
|
|
|
def clone(self, using):
|
|
return RawQuery(self.sql, using, params=self.params)
|
|
|
|
def get_columns(self):
|
|
if self.cursor is None:
|
|
self._execute_query()
|
|
converter = connections[self.using].introspection.identifier_converter
|
|
return [converter(column_meta[0]) for column_meta in self.cursor.description]
|
|
|
|
def __iter__(self):
|
|
# Always execute a new query for a new iterator.
|
|
# This could be optimized with a cache at the expense of RAM.
|
|
self._execute_query()
|
|
if not connections[self.using].features.can_use_chunked_reads:
|
|
# If the database can't use chunked reads we need to make sure we
|
|
# evaluate the entire query up front.
|
|
result = list(self.cursor)
|
|
else:
|
|
result = self.cursor
|
|
return iter(result)
|
|
|
|
def __repr__(self):
|
|
return "<%s: %s>" % (self.__class__.__name__, self)
|
|
|
|
@property
|
|
def params_type(self):
|
|
if self.params is None:
|
|
return None
|
|
return dict if isinstance(self.params, Mapping) else tuple
|
|
|
|
def __str__(self):
|
|
if self.params_type is None:
|
|
return self.sql
|
|
return self.sql % self.params_type(self.params)
|
|
|
|
def _execute_query(self):
|
|
connection = connections[self.using]
|
|
|
|
# Adapt parameters to the database, as much as possible considering
|
|
# that the target type isn't known. See #17755.
|
|
params_type = self.params_type
|
|
adapter = connection.ops.adapt_unknown_value
|
|
if params_type is tuple:
|
|
params = tuple(adapter(val) for val in self.params)
|
|
elif params_type is dict:
|
|
params = {key: adapter(val) for key, val in self.params.items()}
|
|
elif params_type is None:
|
|
params = None
|
|
else:
|
|
raise RuntimeError("Unexpected params type: %s" % params_type)
|
|
|
|
self.cursor = connection.cursor()
|
|
self.cursor.execute(self.sql, params)
|
|
|
|
|
|
ExplainInfo = namedtuple("ExplainInfo", ("format", "options"))
|
|
|
|
|
|
class Query(BaseExpression):
|
|
"""A single SQL query."""
|
|
|
|
alias_prefix = "T"
|
|
empty_result_set_value = None
|
|
subq_aliases = frozenset([alias_prefix])
|
|
|
|
compiler = "SQLCompiler"
|
|
|
|
base_table_class = BaseTable
|
|
join_class = Join
|
|
|
|
default_cols = True
|
|
default_ordering = True
|
|
standard_ordering = True
|
|
|
|
filter_is_sticky = False
|
|
subquery = False
|
|
|
|
# SQL-related attributes.
|
|
# Select and related select clauses are expressions to use in the SELECT
|
|
# clause of the query. The select is used for cases where we want to set up
|
|
# the select clause to contain other than default fields (values(),
|
|
# subqueries...). Note that annotations go to annotations dictionary.
|
|
select = ()
|
|
# The group_by attribute can have one of the following forms:
|
|
# - None: no group by at all in the query
|
|
# - A tuple of expressions: group by (at least) those expressions.
|
|
# String refs are also allowed for now.
|
|
# - True: group by all select fields of the model
|
|
# See compiler.get_group_by() for details.
|
|
group_by = None
|
|
order_by = ()
|
|
low_mark = 0 # Used for offset/limit.
|
|
high_mark = None # Used for offset/limit.
|
|
distinct = False
|
|
distinct_fields = ()
|
|
select_for_update = False
|
|
select_for_update_nowait = False
|
|
select_for_update_skip_locked = False
|
|
select_for_update_of = ()
|
|
select_for_no_key_update = False
|
|
select_related = False
|
|
has_select_fields = False
|
|
# Arbitrary limit for select_related to prevents infinite recursion.
|
|
max_depth = 5
|
|
# Holds the selects defined by a call to values() or values_list()
|
|
# excluding annotation_select and extra_select.
|
|
values_select = ()
|
|
|
|
# SQL annotation-related attributes.
|
|
annotation_select_mask = None
|
|
_annotation_select_cache = None
|
|
|
|
# Set combination attributes.
|
|
combinator = None
|
|
combinator_all = False
|
|
combined_queries = ()
|
|
|
|
# These are for extensions. The contents are more or less appended verbatim
|
|
# to the appropriate clause.
|
|
extra_select_mask = None
|
|
_extra_select_cache = None
|
|
|
|
extra_tables = ()
|
|
extra_order_by = ()
|
|
|
|
# A tuple that is a set of model field names and either True, if these are
|
|
# the fields to defer, or False if these are the only fields to load.
|
|
deferred_loading = (frozenset(), True)
|
|
|
|
explain_info = None
|
|
|
|
def __init__(self, model, alias_cols=True):
|
|
self.model = model
|
|
self.alias_refcount = {}
|
|
# alias_map is the most important data structure regarding joins.
|
|
# It's used for recording which joins exist in the query and what
|
|
# types they are. The key is the alias of the joined table (possibly
|
|
# the table name) and the value is a Join-like object (see
|
|
# sql.datastructures.Join for more information).
|
|
self.alias_map = {}
|
|
# Whether to provide alias to columns during reference resolving.
|
|
self.alias_cols = alias_cols
|
|
# Sometimes the query contains references to aliases in outer queries (as
|
|
# a result of split_exclude). Correct alias quoting needs to know these
|
|
# aliases too.
|
|
# Map external tables to whether they are aliased.
|
|
self.external_aliases = {}
|
|
self.table_map = {} # Maps table names to list of aliases.
|
|
self.used_aliases = set()
|
|
|
|
self.where = WhereNode()
|
|
# Maps alias -> Annotation Expression.
|
|
self.annotations = {}
|
|
# These are for extensions. The contents are more or less appended
|
|
# verbatim to the appropriate clause.
|
|
self.extra = {} # Maps col_alias -> (col_sql, params).
|
|
|
|
self._filtered_relations = {}
|
|
|
|
@property
|
|
def output_field(self):
|
|
if len(self.select) == 1:
|
|
select = self.select[0]
|
|
return getattr(select, "target", None) or select.field
|
|
elif len(self.annotation_select) == 1:
|
|
return next(iter(self.annotation_select.values())).output_field
|
|
|
|
@cached_property
|
|
def base_table(self):
|
|
for alias in self.alias_map:
|
|
return alias
|
|
|
|
def __str__(self):
|
|
"""
|
|
Return the query as a string of SQL with the parameter values
|
|
substituted in (use sql_with_params() to see the unsubstituted string).
|
|
|
|
Parameter values won't necessarily be quoted correctly, since that is
|
|
done by the database interface at execution time.
|
|
"""
|
|
sql, params = self.sql_with_params()
|
|
return sql % params
|
|
|
|
def sql_with_params(self):
|
|
"""
|
|
Return the query as an SQL string and the parameters that will be
|
|
substituted into the query.
|
|
"""
|
|
return self.get_compiler(DEFAULT_DB_ALIAS).as_sql()
|
|
|
|
def __deepcopy__(self, memo):
|
|
"""Limit the amount of work when a Query is deepcopied."""
|
|
result = self.clone()
|
|
memo[id(self)] = result
|
|
return result
|
|
|
|
def get_compiler(self, using=None, connection=None, elide_empty=True):
|
|
if using is None and connection is None:
|
|
raise ValueError("Need either using or connection")
|
|
if using:
|
|
connection = connections[using]
|
|
return connection.ops.compiler(self.compiler)(
|
|
self, connection, using, elide_empty
|
|
)
|
|
|
|
def get_meta(self):
|
|
"""
|
|
Return the Options instance (the model._meta) from which to start
|
|
processing. Normally, this is self.model._meta, but it can be changed
|
|
by subclasses.
|
|
"""
|
|
if self.model:
|
|
return self.model._meta
|
|
|
|
def clone(self):
|
|
"""
|
|
Return a copy of the current Query. A lightweight alternative to
|
|
deepcopy().
|
|
"""
|
|
obj = Empty()
|
|
obj.__class__ = self.__class__
|
|
# Copy references to everything.
|
|
obj.__dict__ = self.__dict__.copy()
|
|
# Clone attributes that can't use shallow copy.
|
|
obj.alias_refcount = self.alias_refcount.copy()
|
|
obj.alias_map = self.alias_map.copy()
|
|
obj.external_aliases = self.external_aliases.copy()
|
|
obj.table_map = self.table_map.copy()
|
|
obj.where = self.where.clone()
|
|
obj.annotations = self.annotations.copy()
|
|
if self.annotation_select_mask is not None:
|
|
obj.annotation_select_mask = self.annotation_select_mask.copy()
|
|
if self.combined_queries:
|
|
obj.combined_queries = tuple(
|
|
[query.clone() for query in self.combined_queries]
|
|
)
|
|
# _annotation_select_cache cannot be copied, as doing so breaks the
|
|
# (necessary) state in which both annotations and
|
|
# _annotation_select_cache point to the same underlying objects.
|
|
# It will get re-populated in the cloned queryset the next time it's
|
|
# used.
|
|
obj._annotation_select_cache = None
|
|
obj.extra = self.extra.copy()
|
|
if self.extra_select_mask is not None:
|
|
obj.extra_select_mask = self.extra_select_mask.copy()
|
|
if self._extra_select_cache is not None:
|
|
obj._extra_select_cache = self._extra_select_cache.copy()
|
|
if self.select_related is not False:
|
|
# Use deepcopy because select_related stores fields in nested
|
|
# dicts.
|
|
obj.select_related = copy.deepcopy(obj.select_related)
|
|
if "subq_aliases" in self.__dict__:
|
|
obj.subq_aliases = self.subq_aliases.copy()
|
|
obj.used_aliases = self.used_aliases.copy()
|
|
obj._filtered_relations = self._filtered_relations.copy()
|
|
# Clear the cached_property, if it exists.
|
|
obj.__dict__.pop("base_table", None)
|
|
return obj
|
|
|
|
def chain(self, klass=None):
|
|
"""
|
|
Return a copy of the current Query that's ready for another operation.
|
|
The klass argument changes the type of the Query, e.g. UpdateQuery.
|
|
"""
|
|
obj = self.clone()
|
|
if klass and obj.__class__ != klass:
|
|
obj.__class__ = klass
|
|
if not obj.filter_is_sticky:
|
|
obj.used_aliases = set()
|
|
obj.filter_is_sticky = False
|
|
if hasattr(obj, "_setup_query"):
|
|
obj._setup_query()
|
|
return obj
|
|
|
|
def relabeled_clone(self, change_map):
|
|
clone = self.clone()
|
|
clone.change_aliases(change_map)
|
|
return clone
|
|
|
|
def _get_col(self, target, field, alias):
|
|
if not self.alias_cols:
|
|
alias = None
|
|
return target.get_col(alias, field)
|
|
|
|
def get_aggregation(self, using, aggregate_exprs):
|
|
"""
|
|
Return the dictionary with the values of the existing aggregations.
|
|
"""
|
|
if not aggregate_exprs:
|
|
return {}
|
|
# Store annotation mask prior to temporarily adding aggregations for
|
|
# resolving purpose to facilitate their subsequent removal.
|
|
refs_subquery = False
|
|
replacements = {}
|
|
annotation_select_mask = self.annotation_select_mask
|
|
for alias, aggregate_expr in aggregate_exprs.items():
|
|
self.check_alias(alias)
|
|
aggregate = aggregate_expr.resolve_expression(
|
|
self, allow_joins=True, reuse=None, summarize=True
|
|
)
|
|
if not aggregate.contains_aggregate:
|
|
raise TypeError("%s is not an aggregate expression" % alias)
|
|
# Temporarily add aggregate to annotations to allow remaining
|
|
# members of `aggregates` to resolve against each others.
|
|
self.append_annotation_mask([alias])
|
|
refs_subquery |= any(
|
|
getattr(self.annotations[ref], "subquery", False)
|
|
for ref in aggregate.get_refs()
|
|
)
|
|
aggregate = aggregate.replace_expressions(replacements)
|
|
self.annotations[alias] = aggregate
|
|
replacements[Ref(alias, aggregate)] = aggregate
|
|
# Stash resolved aggregates now that they have been allowed to resolve
|
|
# against each other.
|
|
aggregates = {alias: self.annotations.pop(alias) for alias in aggregate_exprs}
|
|
self.set_annotation_mask(annotation_select_mask)
|
|
# Existing usage of aggregation can be determined by the presence of
|
|
# selected aggregates but also by filters against aliased aggregates.
|
|
_, having, qualify = self.where.split_having_qualify()
|
|
has_existing_aggregation = (
|
|
any(
|
|
getattr(annotation, "contains_aggregate", True)
|
|
for annotation in self.annotations.values()
|
|
)
|
|
or having
|
|
)
|
|
# Decide if we need to use a subquery.
|
|
#
|
|
# Existing aggregations would cause incorrect results as
|
|
# get_aggregation() must produce just one result and thus must not use
|
|
# GROUP BY.
|
|
#
|
|
# If the query has limit or distinct, or uses set operations, then
|
|
# those operations must be done in a subquery so that the query
|
|
# aggregates on the limit and/or distinct results instead of applying
|
|
# the distinct and limit after the aggregation.
|
|
if (
|
|
isinstance(self.group_by, tuple)
|
|
or self.is_sliced
|
|
or has_existing_aggregation
|
|
or refs_subquery
|
|
or qualify
|
|
or self.distinct
|
|
or self.combinator
|
|
):
|
|
from django.db.models.sql.subqueries import AggregateQuery
|
|
|
|
inner_query = self.clone()
|
|
inner_query.subquery = True
|
|
outer_query = AggregateQuery(self.model, inner_query)
|
|
inner_query.select_for_update = False
|
|
inner_query.select_related = False
|
|
inner_query.set_annotation_mask(self.annotation_select)
|
|
# Queries with distinct_fields need ordering and when a limit is
|
|
# applied we must take the slice from the ordered query. Otherwise
|
|
# no need for ordering.
|
|
inner_query.clear_ordering(force=False)
|
|
if not inner_query.distinct:
|
|
# If the inner query uses default select and it has some
|
|
# aggregate annotations, then we must make sure the inner
|
|
# query is grouped by the main model's primary key. However,
|
|
# clearing the select clause can alter results if distinct is
|
|
# used.
|
|
if inner_query.default_cols and has_existing_aggregation:
|
|
inner_query.group_by = (
|
|
self.model._meta.pk.get_col(inner_query.get_initial_alias()),
|
|
)
|
|
inner_query.default_cols = False
|
|
if not qualify:
|
|
# Mask existing annotations that are not referenced by
|
|
# aggregates to be pushed to the outer query unless
|
|
# filtering against window functions is involved as it
|
|
# requires complex realising.
|
|
annotation_mask = set()
|
|
if isinstance(self.group_by, tuple):
|
|
for expr in self.group_by:
|
|
annotation_mask |= expr.get_refs()
|
|
for aggregate in aggregates.values():
|
|
annotation_mask |= aggregate.get_refs()
|
|
inner_query.set_annotation_mask(annotation_mask)
|
|
|
|
# Add aggregates to the outer AggregateQuery. This requires making
|
|
# sure all columns referenced by the aggregates are selected in the
|
|
# inner query. It is achieved by retrieving all column references
|
|
# by the aggregates, explicitly selecting them in the inner query,
|
|
# and making sure the aggregates are repointed to them.
|
|
col_refs = {}
|
|
for alias, aggregate in aggregates.items():
|
|
replacements = {}
|
|
for col in self._gen_cols([aggregate], resolve_refs=False):
|
|
if not (col_ref := col_refs.get(col)):
|
|
index = len(col_refs) + 1
|
|
col_alias = f"__col{index}"
|
|
col_ref = Ref(col_alias, col)
|
|
col_refs[col] = col_ref
|
|
inner_query.annotations[col_alias] = col
|
|
inner_query.append_annotation_mask([col_alias])
|
|
replacements[col] = col_ref
|
|
outer_query.annotations[alias] = aggregate.replace_expressions(
|
|
replacements
|
|
)
|
|
if (
|
|
inner_query.select == ()
|
|
and not inner_query.default_cols
|
|
and not inner_query.annotation_select_mask
|
|
):
|
|
# In case of Model.objects[0:3].count(), there would be no
|
|
# field selected in the inner query, yet we must use a subquery.
|
|
# So, make sure at least one field is selected.
|
|
inner_query.select = (
|
|
self.model._meta.pk.get_col(inner_query.get_initial_alias()),
|
|
)
|
|
else:
|
|
outer_query = self
|
|
self.select = ()
|
|
self.default_cols = False
|
|
self.extra = {}
|
|
if self.annotations:
|
|
# Inline reference to existing annotations and mask them as
|
|
# they are unnecessary given only the summarized aggregations
|
|
# are requested.
|
|
replacements = {
|
|
Ref(alias, annotation): annotation
|
|
for alias, annotation in self.annotations.items()
|
|
}
|
|
self.annotations = {
|
|
alias: aggregate.replace_expressions(replacements)
|
|
for alias, aggregate in aggregates.items()
|
|
}
|
|
else:
|
|
self.annotations = aggregates
|
|
self.set_annotation_mask(aggregates)
|
|
|
|
empty_set_result = [
|
|
expression.empty_result_set_value
|
|
for expression in outer_query.annotation_select.values()
|
|
]
|
|
elide_empty = not any(result is NotImplemented for result in empty_set_result)
|
|
outer_query.clear_ordering(force=True)
|
|
outer_query.clear_limits()
|
|
outer_query.select_for_update = False
|
|
outer_query.select_related = False
|
|
compiler = outer_query.get_compiler(using, elide_empty=elide_empty)
|
|
result = compiler.execute_sql(SINGLE)
|
|
if result is None:
|
|
result = empty_set_result
|
|
else:
|
|
converters = compiler.get_converters(outer_query.annotation_select.values())
|
|
result = next(compiler.apply_converters((result,), converters))
|
|
|
|
return dict(zip(outer_query.annotation_select, result))
|
|
|
|
def get_count(self, using):
|
|
"""
|
|
Perform a COUNT() query using the current filter constraints.
|
|
"""
|
|
obj = self.clone()
|
|
return obj.get_aggregation(using, {"__count": Count("*")})["__count"]
|
|
|
|
def has_filters(self):
|
|
return self.where
|
|
|
|
def exists(self, limit=True):
|
|
q = self.clone()
|
|
if not (q.distinct and q.is_sliced):
|
|
if q.group_by is True:
|
|
q.add_fields(
|
|
(f.attname for f in self.model._meta.concrete_fields), False
|
|
)
|
|
# Disable GROUP BY aliases to avoid orphaning references to the
|
|
# SELECT clause which is about to be cleared.
|
|
q.set_group_by(allow_aliases=False)
|
|
q.clear_select_clause()
|
|
if q.combined_queries and q.combinator == "union":
|
|
q.combined_queries = tuple(
|
|
combined_query.exists(limit=False)
|
|
for combined_query in q.combined_queries
|
|
)
|
|
q.clear_ordering(force=True)
|
|
if limit:
|
|
q.set_limits(high=1)
|
|
q.add_annotation(Value(1), "a")
|
|
return q
|
|
|
|
def has_results(self, using):
|
|
q = self.exists(using)
|
|
compiler = q.get_compiler(using=using)
|
|
return compiler.has_results()
|
|
|
|
def explain(self, using, format=None, **options):
|
|
q = self.clone()
|
|
for option_name in options:
|
|
if (
|
|
not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name)
|
|
or "--" in option_name
|
|
):
|
|
raise ValueError(f"Invalid option name: {option_name!r}.")
|
|
q.explain_info = ExplainInfo(format, options)
|
|
compiler = q.get_compiler(using=using)
|
|
return "\n".join(compiler.explain_query())
|
|
|
|
def combine(self, rhs, connector):
|
|
"""
|
|
Merge the 'rhs' query into the current one (with any 'rhs' effects
|
|
being applied *after* (that is, "to the right of") anything in the
|
|
current query. 'rhs' is not modified during a call to this function.
|
|
|
|
The 'connector' parameter describes how to connect filters from the
|
|
'rhs' query.
|
|
"""
|
|
if self.model != rhs.model:
|
|
raise TypeError("Cannot combine queries on two different base models.")
|
|
if self.is_sliced:
|
|
raise TypeError("Cannot combine queries once a slice has been taken.")
|
|
if self.distinct != rhs.distinct:
|
|
raise TypeError("Cannot combine a unique query with a non-unique query.")
|
|
if self.distinct_fields != rhs.distinct_fields:
|
|
raise TypeError("Cannot combine queries with different distinct fields.")
|
|
|
|
# If lhs and rhs shares the same alias prefix, it is possible to have
|
|
# conflicting alias changes like T4 -> T5, T5 -> T6, which might end up
|
|
# as T4 -> T6 while combining two querysets. To prevent this, change an
|
|
# alias prefix of the rhs and update current aliases accordingly,
|
|
# except if the alias is the base table since it must be present in the
|
|
# query on both sides.
|
|
initial_alias = self.get_initial_alias()
|
|
rhs.bump_prefix(self, exclude={initial_alias})
|
|
|
|
# Work out how to relabel the rhs aliases, if necessary.
|
|
change_map = {}
|
|
conjunction = connector == AND
|
|
|
|
# Determine which existing joins can be reused. When combining the
|
|
# query with AND we must recreate all joins for m2m filters. When
|
|
# combining with OR we can reuse joins. The reason is that in AND
|
|
# case a single row can't fulfill a condition like:
|
|
# revrel__col=1 & revrel__col=2
|
|
# But, there might be two different related rows matching this
|
|
# condition. In OR case a single True is enough, so single row is
|
|
# enough, too.
|
|
#
|
|
# Note that we will be creating duplicate joins for non-m2m joins in
|
|
# the AND case. The results will be correct but this creates too many
|
|
# joins. This is something that could be fixed later on.
|
|
reuse = set() if conjunction else set(self.alias_map)
|
|
joinpromoter = JoinPromoter(connector, 2, False)
|
|
joinpromoter.add_votes(
|
|
j for j in self.alias_map if self.alias_map[j].join_type == INNER
|
|
)
|
|
rhs_votes = set()
|
|
# Now, add the joins from rhs query into the new query (skipping base
|
|
# table).
|
|
rhs_tables = list(rhs.alias_map)[1:]
|
|
for alias in rhs_tables:
|
|
join = rhs.alias_map[alias]
|
|
# If the left side of the join was already relabeled, use the
|
|
# updated alias.
|
|
join = join.relabeled_clone(change_map)
|
|
new_alias = self.join(join, reuse=reuse)
|
|
if join.join_type == INNER:
|
|
rhs_votes.add(new_alias)
|
|
# We can't reuse the same join again in the query. If we have two
|
|
# distinct joins for the same connection in rhs query, then the
|
|
# combined query must have two joins, too.
|
|
reuse.discard(new_alias)
|
|
if alias != new_alias:
|
|
change_map[alias] = new_alias
|
|
if not rhs.alias_refcount[alias]:
|
|
# The alias was unused in the rhs query. Unref it so that it
|
|
# will be unused in the new query, too. We have to add and
|
|
# unref the alias so that join promotion has information of
|
|
# the join type for the unused alias.
|
|
self.unref_alias(new_alias)
|
|
joinpromoter.add_votes(rhs_votes)
|
|
joinpromoter.update_join_types(self)
|
|
|
|
# Combine subqueries aliases to ensure aliases relabelling properly
|
|
# handle subqueries when combining where and select clauses.
|
|
self.subq_aliases |= rhs.subq_aliases
|
|
|
|
# Now relabel a copy of the rhs where-clause and add it to the current
|
|
# one.
|
|
w = rhs.where.clone()
|
|
w.relabel_aliases(change_map)
|
|
self.where.add(w, connector)
|
|
|
|
# Selection columns and extra extensions are those provided by 'rhs'.
|
|
if rhs.select:
|
|
self.set_select([col.relabeled_clone(change_map) for col in rhs.select])
|
|
else:
|
|
self.select = ()
|
|
|
|
if connector == OR:
|
|
# It would be nice to be able to handle this, but the queries don't
|
|
# really make sense (or return consistent value sets). Not worth
|
|
# the extra complexity when you can write a real query instead.
|
|
if self.extra and rhs.extra:
|
|
raise ValueError(
|
|
"When merging querysets using 'or', you cannot have "
|
|
"extra(select=...) on both sides."
|
|
)
|
|
self.extra.update(rhs.extra)
|
|
extra_select_mask = set()
|
|
if self.extra_select_mask is not None:
|
|
extra_select_mask.update(self.extra_select_mask)
|
|
if rhs.extra_select_mask is not None:
|
|
extra_select_mask.update(rhs.extra_select_mask)
|
|
if extra_select_mask:
|
|
self.set_extra_mask(extra_select_mask)
|
|
self.extra_tables += rhs.extra_tables
|
|
|
|
# Ordering uses the 'rhs' ordering, unless it has none, in which case
|
|
# the current ordering is used.
|
|
self.order_by = rhs.order_by or self.order_by
|
|
self.extra_order_by = rhs.extra_order_by or self.extra_order_by
|
|
|
|
def _get_defer_select_mask(self, opts, mask, select_mask=None):
|
|
if select_mask is None:
|
|
select_mask = {}
|
|
select_mask[opts.pk] = {}
|
|
# All concrete fields that are not part of the defer mask must be
|
|
# loaded. If a relational field is encountered it gets added to the
|
|
# mask for it be considered if `select_related` and the cycle continues
|
|
# by recursively caling this function.
|
|
for field in opts.concrete_fields:
|
|
field_mask = mask.pop(field.name, None)
|
|
field_att_mask = mask.pop(field.attname, None)
|
|
if field_mask is None and field_att_mask is None:
|
|
select_mask.setdefault(field, {})
|
|
elif field_mask:
|
|
if not field.is_relation:
|
|
raise FieldError(next(iter(field_mask)))
|
|
field_select_mask = select_mask.setdefault(field, {})
|
|
related_model = field.remote_field.model._meta.concrete_model
|
|
self._get_defer_select_mask(
|
|
related_model._meta, field_mask, field_select_mask
|
|
)
|
|
# Remaining defer entries must be references to reverse relationships.
|
|
# The following code is expected to raise FieldError if it encounters
|
|
# a malformed defer entry.
|
|
for field_name, field_mask in mask.items():
|
|
if filtered_relation := self._filtered_relations.get(field_name):
|
|
relation = opts.get_field(filtered_relation.relation_name)
|
|
field_select_mask = select_mask.setdefault((field_name, relation), {})
|
|
field = relation.field
|
|
else:
|
|
reverse_rel = opts.get_field(field_name)
|
|
# While virtual fields such as many-to-many and generic foreign
|
|
# keys cannot be effectively deferred we've historically
|
|
# allowed them to be passed to QuerySet.defer(). Ignore such
|
|
# field references until a layer of validation at mask
|
|
# alteration time will be implemented eventually.
|
|
if not hasattr(reverse_rel, "field"):
|
|
continue
|
|
field = reverse_rel.field
|
|
field_select_mask = select_mask.setdefault(field, {})
|
|
related_model = field.model._meta.concrete_model
|
|
self._get_defer_select_mask(
|
|
related_model._meta, field_mask, field_select_mask
|
|
)
|
|
return select_mask
|
|
|
|
def _get_only_select_mask(self, opts, mask, select_mask=None):
|
|
if select_mask is None:
|
|
select_mask = {}
|
|
select_mask[opts.pk] = {}
|
|
# Only include fields mentioned in the mask.
|
|
for field_name, field_mask in mask.items():
|
|
field = opts.get_field(field_name)
|
|
# Retrieve the actual field associated with reverse relationships
|
|
# as that's what is expected in the select mask.
|
|
if field in opts.related_objects:
|
|
field_key = field.field
|
|
else:
|
|
field_key = field
|
|
field_select_mask = select_mask.setdefault(field_key, {})
|
|
if field_mask:
|
|
if not field.is_relation:
|
|
raise FieldError(next(iter(field_mask)))
|
|
related_model = field.remote_field.model._meta.concrete_model
|
|
self._get_only_select_mask(
|
|
related_model._meta, field_mask, field_select_mask
|
|
)
|
|
return select_mask
|
|
|
|
def get_select_mask(self):
|
|
"""
|
|
Convert the self.deferred_loading data structure to an alternate data
|
|
structure, describing the field that *will* be loaded. This is used to
|
|
compute the columns to select from the database and also by the
|
|
QuerySet class to work out which fields are being initialized on each
|
|
model. Models that have all their fields included aren't mentioned in
|
|
the result, only those that have field restrictions in place.
|
|
"""
|
|
field_names, defer = self.deferred_loading
|
|
if not field_names:
|
|
return {}
|
|
mask = {}
|
|
for field_name in field_names:
|
|
part_mask = mask
|
|
for part in field_name.split(LOOKUP_SEP):
|
|
part_mask = part_mask.setdefault(part, {})
|
|
opts = self.get_meta()
|
|
if defer:
|
|
return self._get_defer_select_mask(opts, mask)
|
|
return self._get_only_select_mask(opts, mask)
|
|
|
|
def table_alias(self, table_name, create=False, filtered_relation=None):
|
|
"""
|
|
Return a table alias for the given table_name and whether this is a
|
|
new alias or not.
|
|
|
|
If 'create' is true, a new alias is always created. Otherwise, the
|
|
most recently created alias for the table (if one exists) is reused.
|
|
"""
|
|
alias_list = self.table_map.get(table_name)
|
|
if not create and alias_list:
|
|
alias = alias_list[0]
|
|
self.alias_refcount[alias] += 1
|
|
return alias, False
|
|
|
|
# Create a new alias for this table.
|
|
if alias_list:
|
|
alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1)
|
|
alias_list.append(alias)
|
|
else:
|
|
# The first occurrence of a table uses the table name directly.
|
|
alias = (
|
|
filtered_relation.alias if filtered_relation is not None else table_name
|
|
)
|
|
self.table_map[table_name] = [alias]
|
|
self.alias_refcount[alias] = 1
|
|
return alias, True
|
|
|
|
def ref_alias(self, alias):
|
|
"""Increases the reference count for this alias."""
|
|
self.alias_refcount[alias] += 1
|
|
|
|
def unref_alias(self, alias, amount=1):
|
|
"""Decreases the reference count for this alias."""
|
|
self.alias_refcount[alias] -= amount
|
|
|
|
def promote_joins(self, aliases):
|
|
"""
|
|
Promote recursively the join type of given aliases and its children to
|
|
an outer join. If 'unconditional' is False, only promote the join if
|
|
it is nullable or the parent join is an outer join.
|
|
|
|
The children promotion is done to avoid join chains that contain a LOUTER
|
|
b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted,
|
|
then we must also promote b->c automatically, or otherwise the promotion
|
|
of a->b doesn't actually change anything in the query results.
|
|
"""
|
|
aliases = list(aliases)
|
|
while aliases:
|
|
alias = aliases.pop(0)
|
|
if self.alias_map[alias].join_type is None:
|
|
# This is the base table (first FROM entry) - this table
|
|
# isn't really joined at all in the query, so we should not
|
|
# alter its join type.
|
|
continue
|
|
# Only the first alias (skipped above) should have None join_type
|
|
assert self.alias_map[alias].join_type is not None
|
|
parent_alias = self.alias_map[alias].parent_alias
|
|
parent_louter = (
|
|
parent_alias and self.alias_map[parent_alias].join_type == LOUTER
|
|
)
|
|
already_louter = self.alias_map[alias].join_type == LOUTER
|
|
if (self.alias_map[alias].nullable or parent_louter) and not already_louter:
|
|
self.alias_map[alias] = self.alias_map[alias].promote()
|
|
# Join type of 'alias' changed, so re-examine all aliases that
|
|
# refer to this one.
|
|
aliases.extend(
|
|
join
|
|
for join in self.alias_map
|
|
if self.alias_map[join].parent_alias == alias
|
|
and join not in aliases
|
|
)
|
|
|
|
def demote_joins(self, aliases):
|
|
"""
|
|
Change join type from LOUTER to INNER for all joins in aliases.
|
|
|
|
Similarly to promote_joins(), this method must ensure no join chains
|
|
containing first an outer, then an inner join are generated. If we
|
|
are demoting b->c join in chain a LOUTER b LOUTER c then we must
|
|
demote a->b automatically, or otherwise the demotion of b->c doesn't
|
|
actually change anything in the query results. .
|
|
"""
|
|
aliases = list(aliases)
|
|
while aliases:
|
|
alias = aliases.pop(0)
|
|
if self.alias_map[alias].join_type == LOUTER:
|
|
self.alias_map[alias] = self.alias_map[alias].demote()
|
|
parent_alias = self.alias_map[alias].parent_alias
|
|
if self.alias_map[parent_alias].join_type == INNER:
|
|
aliases.append(parent_alias)
|
|
|
|
def reset_refcounts(self, to_counts):
|
|
"""
|
|
Reset reference counts for aliases so that they match the value passed
|
|
in `to_counts`.
|
|
"""
|
|
for alias, cur_refcount in self.alias_refcount.copy().items():
|
|
unref_amount = cur_refcount - to_counts.get(alias, 0)
|
|
self.unref_alias(alias, unref_amount)
|
|
|
|
def change_aliases(self, change_map):
|
|
"""
|
|
Change the aliases in change_map (which maps old-alias -> new-alias),
|
|
relabelling any references to them in select columns and the where
|
|
clause.
|
|
"""
|
|
# If keys and values of change_map were to intersect, an alias might be
|
|
# updated twice (e.g. T4 -> T5, T5 -> T6, so also T4 -> T6) depending
|
|
# on their order in change_map.
|
|
assert set(change_map).isdisjoint(change_map.values())
|
|
|
|
# 1. Update references in "select" (normal columns plus aliases),
|
|
# "group by" and "where".
|
|
self.where.relabel_aliases(change_map)
|
|
if isinstance(self.group_by, tuple):
|
|
self.group_by = tuple(
|
|
[col.relabeled_clone(change_map) for col in self.group_by]
|
|
)
|
|
self.select = tuple([col.relabeled_clone(change_map) for col in self.select])
|
|
self.annotations = self.annotations and {
|
|
key: col.relabeled_clone(change_map)
|
|
for key, col in self.annotations.items()
|
|
}
|
|
|
|
# 2. Rename the alias in the internal table/alias datastructures.
|
|
for old_alias, new_alias in change_map.items():
|
|
if old_alias not in self.alias_map:
|
|
continue
|
|
alias_data = self.alias_map[old_alias].relabeled_clone(change_map)
|
|
self.alias_map[new_alias] = alias_data
|
|
self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
|
|
del self.alias_refcount[old_alias]
|
|
del self.alias_map[old_alias]
|
|
|
|
table_aliases = self.table_map[alias_data.table_name]
|
|
for pos, alias in enumerate(table_aliases):
|
|
if alias == old_alias:
|
|
table_aliases[pos] = new_alias
|
|
break
|
|
self.external_aliases = {
|
|
# Table is aliased or it's being changed and thus is aliased.
|
|
change_map.get(alias, alias): (aliased or alias in change_map)
|
|
for alias, aliased in self.external_aliases.items()
|
|
}
|
|
|
|
def bump_prefix(self, other_query, exclude=None):
|
|
"""
|
|
Change the alias prefix to the next letter in the alphabet in a way
|
|
that the other query's aliases and this query's aliases will not
|
|
conflict. Even tables that previously had no alias will get an alias
|
|
after this call. To prevent changing aliases use the exclude parameter.
|
|
"""
|
|
|
|
def prefix_gen():
|
|
"""
|
|
Generate a sequence of characters in alphabetical order:
|
|
-> 'A', 'B', 'C', ...
|
|
|
|
When the alphabet is finished, the sequence will continue with the
|
|
Cartesian product:
|
|
-> 'AA', 'AB', 'AC', ...
|
|
"""
|
|
alphabet = ascii_uppercase
|
|
prefix = chr(ord(self.alias_prefix) + 1)
|
|
yield prefix
|
|
for n in count(1):
|
|
seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet
|
|
for s in product(seq, repeat=n):
|
|
yield "".join(s)
|
|
prefix = None
|
|
|
|
if self.alias_prefix != other_query.alias_prefix:
|
|
# No clashes between self and outer query should be possible.
|
|
return
|
|
|
|
# Explicitly avoid infinite loop. The constant divider is based on how
|
|
# much depth recursive subquery references add to the stack. This value
|
|
# might need to be adjusted when adding or removing function calls from
|
|
# the code path in charge of performing these operations.
|
|
local_recursion_limit = sys.getrecursionlimit() // 16
|
|
for pos, prefix in enumerate(prefix_gen()):
|
|
if prefix not in self.subq_aliases:
|
|
self.alias_prefix = prefix
|
|
break
|
|
if pos > local_recursion_limit:
|
|
raise RecursionError(
|
|
"Maximum recursion depth exceeded: too many subqueries."
|
|
)
|
|
self.subq_aliases = self.subq_aliases.union([self.alias_prefix])
|
|
other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases)
|
|
if exclude is None:
|
|
exclude = {}
|
|
self.change_aliases(
|
|
{
|
|
alias: "%s%d" % (self.alias_prefix, pos)
|
|
for pos, alias in enumerate(self.alias_map)
|
|
if alias not in exclude
|
|
}
|
|
)
|
|
|
|
def get_initial_alias(self):
|
|
"""
|
|
Return the first alias for this query, after increasing its reference
|
|
count.
|
|
"""
|
|
if self.alias_map:
|
|
alias = self.base_table
|
|
self.ref_alias(alias)
|
|
elif self.model:
|
|
alias = self.join(self.base_table_class(self.get_meta().db_table, None))
|
|
else:
|
|
alias = None
|
|
return alias
|
|
|
|
def count_active_tables(self):
|
|
"""
|
|
Return the number of tables in this query with a non-zero reference
|
|
count. After execution, the reference counts are zeroed, so tables
|
|
added in compiler will not be seen by this method.
|
|
"""
|
|
return len([1 for count in self.alias_refcount.values() if count])
|
|
|
|
def join(self, join, reuse=None, reuse_with_filtered_relation=False):
|
|
"""
|
|
Return an alias for the 'join', either reusing an existing alias for
|
|
that join or creating a new one. 'join' is either a base_table_class or
|
|
join_class.
|
|
|
|
The 'reuse' parameter can be either None which means all joins are
|
|
reusable, or it can be a set containing the aliases that can be reused.
|
|
|
|
The 'reuse_with_filtered_relation' parameter is used when computing
|
|
FilteredRelation instances.
|
|
|
|
A join is always created as LOUTER if the lhs alias is LOUTER to make
|
|
sure chains like t1 LOUTER t2 INNER t3 aren't generated. All new
|
|
joins are created as LOUTER if the join is nullable.
|
|
"""
|
|
if reuse_with_filtered_relation and reuse:
|
|
reuse_aliases = [
|
|
a for a, j in self.alias_map.items() if a in reuse and j.equals(join)
|
|
]
|
|
else:
|
|
reuse_aliases = [
|
|
a
|
|
for a, j in self.alias_map.items()
|
|
if (reuse is None or a in reuse) and j == join
|
|
]
|
|
if reuse_aliases:
|
|
if join.table_alias in reuse_aliases:
|
|
reuse_alias = join.table_alias
|
|
else:
|
|
# Reuse the most recent alias of the joined table
|
|
# (a many-to-many relation may be joined multiple times).
|
|
reuse_alias = reuse_aliases[-1]
|
|
self.ref_alias(reuse_alias)
|
|
return reuse_alias
|
|
|
|
# No reuse is possible, so we need a new alias.
|
|
alias, _ = self.table_alias(
|
|
join.table_name, create=True, filtered_relation=join.filtered_relation
|
|
)
|
|
if join.join_type:
|
|
if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable:
|
|
join_type = LOUTER
|
|
else:
|
|
join_type = INNER
|
|
join.join_type = join_type
|
|
join.table_alias = alias
|
|
self.alias_map[alias] = join
|
|
return alias
|
|
|
|
def join_parent_model(self, opts, model, alias, seen):
|
|
"""
|
|
Make sure the given 'model' is joined in the query. If 'model' isn't
|
|
a parent of 'opts' or if it is None this method is a no-op.
|
|
|
|
The 'alias' is the root alias for starting the join, 'seen' is a dict
|
|
of model -> alias of existing joins. It must also contain a mapping
|
|
of None -> some alias. This will be returned in the no-op case.
|
|
"""
|
|
if model in seen:
|
|
return seen[model]
|
|
chain = opts.get_base_chain(model)
|
|
if not chain:
|
|
return alias
|
|
curr_opts = opts
|
|
for int_model in chain:
|
|
if int_model in seen:
|
|
curr_opts = int_model._meta
|
|
alias = seen[int_model]
|
|
continue
|
|
# Proxy model have elements in base chain
|
|
# with no parents, assign the new options
|
|
# object and skip to the next base in that
|
|
# case
|
|
if not curr_opts.parents[int_model]:
|
|
curr_opts = int_model._meta
|
|
continue
|
|
link_field = curr_opts.get_ancestor_link(int_model)
|
|
join_info = self.setup_joins([link_field.name], curr_opts, alias)
|
|
curr_opts = int_model._meta
|
|
alias = seen[int_model] = join_info.joins[-1]
|
|
return alias or seen[None]
|
|
|
|
def check_alias(self, alias):
|
|
if FORBIDDEN_ALIAS_PATTERN.search(alias):
|
|
raise ValueError(
|
|
"Column aliases cannot contain whitespace characters, quotation marks, "
|
|
"semicolons, or SQL comments."
|
|
)
|
|
|
|
def add_annotation(self, annotation, alias, select=True):
|
|
"""Add a single annotation expression to the Query."""
|
|
self.check_alias(alias)
|
|
annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None)
|
|
if select:
|
|
self.append_annotation_mask([alias])
|
|
else:
|
|
self.set_annotation_mask(set(self.annotation_select).difference({alias}))
|
|
self.annotations[alias] = annotation
|
|
|
|
def resolve_expression(self, query, *args, **kwargs):
|
|
clone = self.clone()
|
|
# Subqueries need to use a different set of aliases than the outer query.
|
|
clone.bump_prefix(query)
|
|
clone.subquery = True
|
|
clone.where.resolve_expression(query, *args, **kwargs)
|
|
# Resolve combined queries.
|
|
if clone.combinator:
|
|
clone.combined_queries = tuple(
|
|
[
|
|
combined_query.resolve_expression(query, *args, **kwargs)
|
|
for combined_query in clone.combined_queries
|
|
]
|
|
)
|
|
for key, value in clone.annotations.items():
|
|
resolved = value.resolve_expression(query, *args, **kwargs)
|
|
if hasattr(resolved, "external_aliases"):
|
|
resolved.external_aliases.update(clone.external_aliases)
|
|
clone.annotations[key] = resolved
|
|
# Outer query's aliases are considered external.
|
|
for alias, table in query.alias_map.items():
|
|
clone.external_aliases[alias] = (
|
|
isinstance(table, Join)
|
|
and table.join_field.related_model._meta.db_table != alias
|
|
) or (
|
|
isinstance(table, BaseTable) and table.table_name != table.table_alias
|
|
)
|
|
return clone
|
|
|
|
def get_external_cols(self):
|
|
exprs = chain(self.annotations.values(), self.where.children)
|
|
return [
|
|
col
|
|
for col in self._gen_cols(exprs, include_external=True)
|
|
if col.alias in self.external_aliases
|
|
]
|
|
|
|
def get_group_by_cols(self, wrapper=None):
|
|
# If wrapper is referenced by an alias for an explicit GROUP BY through
|
|
# values() a reference to this expression and not the self must be
|
|
# returned to ensure external column references are not grouped against
|
|
# as well.
|
|
external_cols = self.get_external_cols()
|
|
if any(col.possibly_multivalued for col in external_cols):
|
|
return [wrapper or self]
|
|
return external_cols
|
|
|
|
def as_sql(self, compiler, connection):
|
|
# Some backends (e.g. Oracle) raise an error when a subquery contains
|
|
# unnecessary ORDER BY clause.
|
|
if (
|
|
self.subquery
|
|
and not connection.features.ignores_unnecessary_order_by_in_subqueries
|
|
):
|
|
self.clear_ordering(force=False)
|
|
for query in self.combined_queries:
|
|
query.clear_ordering(force=False)
|
|
sql, params = self.get_compiler(connection=connection).as_sql()
|
|
if self.subquery:
|
|
sql = "(%s)" % sql
|
|
return sql, params
|
|
|
|
def resolve_lookup_value(self, value, can_reuse, allow_joins):
|
|
if hasattr(value, "resolve_expression"):
|
|
value = value.resolve_expression(
|
|
self,
|
|
reuse=can_reuse,
|
|
allow_joins=allow_joins,
|
|
)
|
|
elif isinstance(value, (list, tuple)):
|
|
# The items of the iterable may be expressions and therefore need
|
|
# to be resolved independently.
|
|
values = (
|
|
self.resolve_lookup_value(sub_value, can_reuse, allow_joins)
|
|
for sub_value in value
|
|
)
|
|
type_ = type(value)
|
|
if hasattr(type_, "_make"): # namedtuple
|
|
return type_(*values)
|
|
return type_(values)
|
|
return value
|
|
|
|
def solve_lookup_type(self, lookup, summarize=False):
|
|
"""
|
|
Solve the lookup type from the lookup (e.g.: 'foobar__id__icontains').
|
|
"""
|
|
lookup_splitted = lookup.split(LOOKUP_SEP)
|
|
if self.annotations:
|
|
annotation, expression_lookups = refs_expression(
|
|
lookup_splitted, self.annotations
|
|
)
|
|
if annotation:
|
|
expression = self.annotations[annotation]
|
|
if summarize:
|
|
expression = Ref(annotation, expression)
|
|
return expression_lookups, (), expression
|
|
_, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
|
|
field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)]
|
|
if len(lookup_parts) > 1 and not field_parts:
|
|
raise FieldError(
|
|
'Invalid lookup "%s" for model %s".'
|
|
% (lookup, self.get_meta().model.__name__)
|
|
)
|
|
return lookup_parts, field_parts, False
|
|
|
|
def check_query_object_type(self, value, opts, field):
|
|
"""
|
|
Check whether the object passed while querying is of the correct type.
|
|
If not, raise a ValueError specifying the wrong object.
|
|
"""
|
|
if hasattr(value, "_meta"):
|
|
if not check_rel_lookup_compatibility(value._meta.model, opts, field):
|
|
raise ValueError(
|
|
'Cannot query "%s": Must be "%s" instance.'
|
|
% (value, opts.object_name)
|
|
)
|
|
|
|
def check_related_objects(self, field, value, opts):
|
|
"""Check the type of object passed to query relations."""
|
|
if field.is_relation:
|
|
# Check that the field and the queryset use the same model in a
|
|
# query like .filter(author=Author.objects.all()). For example, the
|
|
# opts would be Author's (from the author field) and value.model
|
|
# would be Author.objects.all() queryset's .model (Author also).
|
|
# The field is the related field on the lhs side.
|
|
if (
|
|
isinstance(value, Query)
|
|
and not value.has_select_fields
|
|
and not check_rel_lookup_compatibility(value.model, opts, field)
|
|
):
|
|
raise ValueError(
|
|
'Cannot use QuerySet for "%s": Use a QuerySet for "%s".'
|
|
% (value.model._meta.object_name, opts.object_name)
|
|
)
|
|
elif hasattr(value, "_meta"):
|
|
self.check_query_object_type(value, opts, field)
|
|
elif hasattr(value, "__iter__"):
|
|
for v in value:
|
|
self.check_query_object_type(v, opts, field)
|
|
|
|
def check_filterable(self, expression):
|
|
"""Raise an error if expression cannot be used in a WHERE clause."""
|
|
if hasattr(expression, "resolve_expression") and not getattr(
|
|
expression, "filterable", True
|
|
):
|
|
raise NotSupportedError(
|
|
expression.__class__.__name__ + " is disallowed in the filter "
|
|
"clause."
|
|
)
|
|
if hasattr(expression, "get_source_expressions"):
|
|
for expr in expression.get_source_expressions():
|
|
self.check_filterable(expr)
|
|
|
|
def build_lookup(self, lookups, lhs, rhs):
|
|
"""
|
|
Try to extract transforms and lookup from given lhs.
|
|
|
|
The lhs value is something that works like SQLExpression.
|
|
The rhs value is what the lookup is going to compare against.
|
|
The lookups is a list of names to extract using get_lookup()
|
|
and get_transform().
|
|
"""
|
|
# __exact is the default lookup if one isn't given.
|
|
*transforms, lookup_name = lookups or ["exact"]
|
|
for name in transforms:
|
|
lhs = self.try_transform(lhs, name)
|
|
# First try get_lookup() so that the lookup takes precedence if the lhs
|
|
# supports both transform and lookup for the name.
|
|
lookup_class = lhs.get_lookup(lookup_name)
|
|
if not lookup_class:
|
|
# A lookup wasn't found. Try to interpret the name as a transform
|
|
# and do an Exact lookup against it.
|
|
lhs = self.try_transform(lhs, lookup_name)
|
|
lookup_name = "exact"
|
|
lookup_class = lhs.get_lookup(lookup_name)
|
|
if not lookup_class:
|
|
return
|
|
|
|
lookup = lookup_class(lhs, rhs)
|
|
# Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all
|
|
# uses of None as a query value unless the lookup supports it.
|
|
if lookup.rhs is None and not lookup.can_use_none_as_rhs:
|
|
if lookup_name not in ("exact", "iexact"):
|
|
raise ValueError("Cannot use None as a query value")
|
|
return lhs.get_lookup("isnull")(lhs, True)
|
|
|
|
# For Oracle '' is equivalent to null. The check must be done at this
|
|
# stage because join promotion can't be done in the compiler. Using
|
|
# DEFAULT_DB_ALIAS isn't nice but it's the best that can be done here.
|
|
# A similar thing is done in is_nullable(), too.
|
|
if (
|
|
lookup_name == "exact"
|
|
and lookup.rhs == ""
|
|
and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
|
|
):
|
|
return lhs.get_lookup("isnull")(lhs, True)
|
|
|
|
return lookup
|
|
|
|
def try_transform(self, lhs, name):
|
|
"""
|
|
Helper method for build_lookup(). Try to fetch and initialize
|
|
a transform for name parameter from lhs.
|
|
"""
|
|
transform_class = lhs.get_transform(name)
|
|
if transform_class:
|
|
return transform_class(lhs)
|
|
else:
|
|
output_field = lhs.output_field.__class__
|
|
suggested_lookups = difflib.get_close_matches(
|
|
name, lhs.output_field.get_lookups()
|
|
)
|
|
if suggested_lookups:
|
|
suggestion = ", perhaps you meant %s?" % " or ".join(suggested_lookups)
|
|
else:
|
|
suggestion = "."
|
|
raise FieldError(
|
|
"Unsupported lookup '%s' for %s or join on the field not "
|
|
"permitted%s" % (name, output_field.__name__, suggestion)
|
|
)
|
|
|
|
def build_filter(
|
|
self,
|
|
filter_expr,
|
|
branch_negated=False,
|
|
current_negated=False,
|
|
can_reuse=None,
|
|
allow_joins=True,
|
|
split_subq=True,
|
|
reuse_with_filtered_relation=False,
|
|
check_filterable=True,
|
|
summarize=False,
|
|
):
|
|
"""
|
|
Build a WhereNode for a single filter clause but don't add it
|
|
to this Query. Query.add_q() will then add this filter to the where
|
|
Node.
|
|
|
|
The 'branch_negated' tells us if the current branch contains any
|
|
negations. This will be used to determine if subqueries are needed.
|
|
|
|
The 'current_negated' is used to determine if the current filter is
|
|
negated or not and this will be used to determine if IS NULL filtering
|
|
is needed.
|
|
|
|
The difference between current_negated and branch_negated is that
|
|
branch_negated is set on first negation, but current_negated is
|
|
flipped for each negation.
|
|
|
|
Note that add_filter will not do any negating itself, that is done
|
|
upper in the code by add_q().
|
|
|
|
The 'can_reuse' is a set of reusable joins for multijoins.
|
|
|
|
If 'reuse_with_filtered_relation' is True, then only joins in can_reuse
|
|
will be reused.
|
|
|
|
The method will create a filter clause that can be added to the current
|
|
query. However, if the filter isn't added to the query then the caller
|
|
is responsible for unreffing the joins used.
|
|
"""
|
|
if isinstance(filter_expr, dict):
|
|
raise FieldError("Cannot parse keyword query as dict")
|
|
if isinstance(filter_expr, Q):
|
|
return self._add_q(
|
|
filter_expr,
|
|
branch_negated=branch_negated,
|
|
current_negated=current_negated,
|
|
used_aliases=can_reuse,
|
|
allow_joins=allow_joins,
|
|
split_subq=split_subq,
|
|
check_filterable=check_filterable,
|
|
summarize=summarize,
|
|
)
|
|
if hasattr(filter_expr, "resolve_expression"):
|
|
if not getattr(filter_expr, "conditional", False):
|
|
raise TypeError("Cannot filter against a non-conditional expression.")
|
|
condition = filter_expr.resolve_expression(
|
|
self, allow_joins=allow_joins, summarize=summarize
|
|
)
|
|
if not isinstance(condition, Lookup):
|
|
condition = self.build_lookup(["exact"], condition, True)
|
|
return WhereNode([condition], connector=AND), []
|
|
arg, value = filter_expr
|
|
if not arg:
|
|
raise FieldError("Cannot parse keyword query %r" % arg)
|
|
lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize)
|
|
|
|
if check_filterable:
|
|
self.check_filterable(reffed_expression)
|
|
|
|
if not allow_joins and len(parts) > 1:
|
|
raise FieldError("Joined field references are not permitted in this query")
|
|
|
|
pre_joins = self.alias_refcount.copy()
|
|
value = self.resolve_lookup_value(value, can_reuse, allow_joins)
|
|
used_joins = {
|
|
k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)
|
|
}
|
|
|
|
if check_filterable:
|
|
self.check_filterable(value)
|
|
|
|
if reffed_expression:
|
|
condition = self.build_lookup(lookups, reffed_expression, value)
|
|
return WhereNode([condition], connector=AND), []
|
|
|
|
opts = self.get_meta()
|
|
alias = self.get_initial_alias()
|
|
allow_many = not branch_negated or not split_subq
|
|
|
|
try:
|
|
join_info = self.setup_joins(
|
|
parts,
|
|
opts,
|
|
alias,
|
|
can_reuse=can_reuse,
|
|
allow_many=allow_many,
|
|
reuse_with_filtered_relation=reuse_with_filtered_relation,
|
|
)
|
|
|
|
# Prevent iterator from being consumed by check_related_objects()
|
|
if isinstance(value, Iterator):
|
|
value = list(value)
|
|
self.check_related_objects(join_info.final_field, value, join_info.opts)
|
|
|
|
# split_exclude() needs to know which joins were generated for the
|
|
# lookup parts
|
|
self._lookup_joins = join_info.joins
|
|
except MultiJoin as e:
|
|
return self.split_exclude(filter_expr, can_reuse, e.names_with_path)
|
|
|
|
# Update used_joins before trimming since they are reused to determine
|
|
# which joins could be later promoted to INNER.
|
|
used_joins.update(join_info.joins)
|
|
targets, alias, join_list = self.trim_joins(
|
|
join_info.targets, join_info.joins, join_info.path
|
|
)
|
|
if can_reuse is not None:
|
|
can_reuse.update(join_list)
|
|
|
|
if join_info.final_field.is_relation:
|
|
if len(targets) == 1:
|
|
col = self._get_col(targets[0], join_info.final_field, alias)
|
|
else:
|
|
col = MultiColSource(
|
|
alias, targets, join_info.targets, join_info.final_field
|
|
)
|
|
else:
|
|
col = self._get_col(targets[0], join_info.final_field, alias)
|
|
|
|
condition = self.build_lookup(lookups, col, value)
|
|
lookup_type = condition.lookup_name
|
|
clause = WhereNode([condition], connector=AND)
|
|
|
|
require_outer = (
|
|
lookup_type == "isnull" and condition.rhs is True and not current_negated
|
|
)
|
|
if (
|
|
current_negated
|
|
and (lookup_type != "isnull" or condition.rhs is False)
|
|
and condition.rhs is not None
|
|
):
|
|
require_outer = True
|
|
if lookup_type != "isnull":
|
|
# The condition added here will be SQL like this:
|
|
# NOT (col IS NOT NULL), where the first NOT is added in
|
|
# upper layers of code. The reason for addition is that if col
|
|
# is null, then col != someval will result in SQL "unknown"
|
|
# which isn't the same as in Python. The Python None handling
|
|
# is wanted, and it can be gotten by
|
|
# (col IS NULL OR col != someval)
|
|
# <=>
|
|
# NOT (col IS NOT NULL AND col = someval).
|
|
if (
|
|
self.is_nullable(targets[0])
|
|
or self.alias_map[join_list[-1]].join_type == LOUTER
|
|
):
|
|
lookup_class = targets[0].get_lookup("isnull")
|
|
col = self._get_col(targets[0], join_info.targets[0], alias)
|
|
clause.add(lookup_class(col, False), AND)
|
|
# If someval is a nullable column, someval IS NOT NULL is
|
|
# added.
|
|
if isinstance(value, Col) and self.is_nullable(value.target):
|
|
lookup_class = value.target.get_lookup("isnull")
|
|
clause.add(lookup_class(value, False), AND)
|
|
return clause, used_joins if not require_outer else ()
|
|
|
|
def add_filter(self, filter_lhs, filter_rhs):
|
|
self.add_q(Q((filter_lhs, filter_rhs)))
|
|
|
|
def add_q(self, q_object):
|
|
"""
|
|
A preprocessor for the internal _add_q(). Responsible for doing final
|
|
join promotion.
|
|
"""
|
|
# For join promotion this case is doing an AND for the added q_object
|
|
# and existing conditions. So, any existing inner join forces the join
|
|
# type to remain inner. Existing outer joins can however be demoted.
|
|
# (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if
|
|
# rel_a doesn't produce any rows, then the whole condition must fail.
|
|
# So, demotion is OK.
|
|
existing_inner = {
|
|
a for a in self.alias_map if self.alias_map[a].join_type == INNER
|
|
}
|
|
clause, _ = self._add_q(q_object, self.used_aliases)
|
|
if clause:
|
|
self.where.add(clause, AND)
|
|
self.demote_joins(existing_inner)
|
|
|
|
def build_where(self, filter_expr):
|
|
return self.build_filter(filter_expr, allow_joins=False)[0]
|
|
|
|
def clear_where(self):
|
|
self.where = WhereNode()
|
|
|
|
def _add_q(
|
|
self,
|
|
q_object,
|
|
used_aliases,
|
|
branch_negated=False,
|
|
current_negated=False,
|
|
allow_joins=True,
|
|
split_subq=True,
|
|
check_filterable=True,
|
|
summarize=False,
|
|
):
|
|
"""Add a Q-object to the current filter."""
|
|
connector = q_object.connector
|
|
current_negated ^= q_object.negated
|
|
branch_negated = branch_negated or q_object.negated
|
|
target_clause = WhereNode(connector=connector, negated=q_object.negated)
|
|
joinpromoter = JoinPromoter(
|
|
q_object.connector, len(q_object.children), current_negated
|
|
)
|
|
for child in q_object.children:
|
|
child_clause, needed_inner = self.build_filter(
|
|
child,
|
|
can_reuse=used_aliases,
|
|
branch_negated=branch_negated,
|
|
current_negated=current_negated,
|
|
allow_joins=allow_joins,
|
|
split_subq=split_subq,
|
|
check_filterable=check_filterable,
|
|
summarize=summarize,
|
|
)
|
|
joinpromoter.add_votes(needed_inner)
|
|
if child_clause:
|
|
target_clause.add(child_clause, connector)
|
|
needed_inner = joinpromoter.update_join_types(self)
|
|
return target_clause, needed_inner
|
|
|
|
def build_filtered_relation_q(
|
|
self, q_object, reuse, branch_negated=False, current_negated=False
|
|
):
|
|
"""Add a FilteredRelation object to the current filter."""
|
|
connector = q_object.connector
|
|
current_negated ^= q_object.negated
|
|
branch_negated = branch_negated or q_object.negated
|
|
target_clause = WhereNode(connector=connector, negated=q_object.negated)
|
|
for child in q_object.children:
|
|
if isinstance(child, Node):
|
|
child_clause = self.build_filtered_relation_q(
|
|
child,
|
|
reuse=reuse,
|
|
branch_negated=branch_negated,
|
|
current_negated=current_negated,
|
|
)
|
|
else:
|
|
child_clause, _ = self.build_filter(
|
|
child,
|
|
can_reuse=reuse,
|
|
branch_negated=branch_negated,
|
|
current_negated=current_negated,
|
|
allow_joins=True,
|
|
split_subq=False,
|
|
reuse_with_filtered_relation=True,
|
|
)
|
|
target_clause.add(child_clause, connector)
|
|
return target_clause
|
|
|
|
def add_filtered_relation(self, filtered_relation, alias):
|
|
filtered_relation.alias = alias
|
|
lookups = dict(get_children_from_q(filtered_relation.condition))
|
|
relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type(
|
|
filtered_relation.relation_name
|
|
)
|
|
if relation_lookup_parts:
|
|
raise ValueError(
|
|
"FilteredRelation's relation_name cannot contain lookups "
|
|
"(got %r)." % filtered_relation.relation_name
|
|
)
|
|
for lookup in chain(lookups):
|
|
lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup)
|
|
shift = 2 if not lookup_parts else 1
|
|
lookup_field_path = lookup_field_parts[:-shift]
|
|
for idx, lookup_field_part in enumerate(lookup_field_path):
|
|
if len(relation_field_parts) > idx:
|
|
if relation_field_parts[idx] != lookup_field_part:
|
|
raise ValueError(
|
|
"FilteredRelation's condition doesn't support "
|
|
"relations outside the %r (got %r)."
|
|
% (filtered_relation.relation_name, lookup)
|
|
)
|
|
else:
|
|
raise ValueError(
|
|
"FilteredRelation's condition doesn't support nested "
|
|
"relations deeper than the relation_name (got %r for "
|
|
"%r)." % (lookup, filtered_relation.relation_name)
|
|
)
|
|
self._filtered_relations[filtered_relation.alias] = filtered_relation
|
|
|
|
def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False):
|
|
"""
|
|
Walk the list of names and turns them into PathInfo tuples. A single
|
|
name in 'names' can generate multiple PathInfos (m2m, for example).
|
|
|
|
'names' is the path of names to travel, 'opts' is the model Options we
|
|
start the name resolving from, 'allow_many' is as for setup_joins().
|
|
If fail_on_missing is set to True, then a name that can't be resolved
|
|
will generate a FieldError.
|
|
|
|
Return a list of PathInfo tuples. In addition return the final field
|
|
(the last used join field) and target (which is a field guaranteed to
|
|
contain the same value as the final field). Finally, return those names
|
|
that weren't found (which are likely transforms and the final lookup).
|
|
"""
|
|
path, names_with_path = [], []
|
|
for pos, name in enumerate(names):
|
|
cur_names_with_path = (name, [])
|
|
if name == "pk":
|
|
name = opts.pk.name
|
|
|
|
field = None
|
|
filtered_relation = None
|
|
try:
|
|
if opts is None:
|
|
raise FieldDoesNotExist
|
|
field = opts.get_field(name)
|
|
except FieldDoesNotExist:
|
|
if name in self.annotation_select:
|
|
field = self.annotation_select[name].output_field
|
|
elif name in self._filtered_relations and pos == 0:
|
|
filtered_relation = self._filtered_relations[name]
|
|
if LOOKUP_SEP in filtered_relation.relation_name:
|
|
parts = filtered_relation.relation_name.split(LOOKUP_SEP)
|
|
filtered_relation_path, field, _, _ = self.names_to_path(
|
|
parts,
|
|
opts,
|
|
allow_many,
|
|
fail_on_missing,
|
|
)
|
|
path.extend(filtered_relation_path[:-1])
|
|
else:
|
|
field = opts.get_field(filtered_relation.relation_name)
|
|
if field is not None:
|
|
# Fields that contain one-to-many relations with a generic
|
|
# model (like a GenericForeignKey) cannot generate reverse
|
|
# relations and therefore cannot be used for reverse querying.
|
|
if field.is_relation and not field.related_model:
|
|
raise FieldError(
|
|
"Field %r does not generate an automatic reverse "
|
|
"relation and therefore cannot be used for reverse "
|
|
"querying. If it is a GenericForeignKey, consider "
|
|
"adding a GenericRelation." % name
|
|
)
|
|
try:
|
|
model = field.model._meta.concrete_model
|
|
except AttributeError:
|
|
# QuerySet.annotate() may introduce fields that aren't
|
|
# attached to a model.
|
|
model = None
|
|
else:
|
|
# We didn't find the current field, so move position back
|
|
# one step.
|
|
pos -= 1
|
|
if pos == -1 or fail_on_missing:
|
|
available = sorted(
|
|
[
|
|
*get_field_names_from_opts(opts),
|
|
*self.annotation_select,
|
|
*self._filtered_relations,
|
|
]
|
|
)
|
|
raise FieldError(
|
|
"Cannot resolve keyword '%s' into field. "
|
|
"Choices are: %s" % (name, ", ".join(available))
|
|
)
|
|
break
|
|
# Check if we need any joins for concrete inheritance cases (the
|
|
# field lives in parent, but we are currently in one of its
|
|
# children)
|
|
if opts is not None and model is not opts.model:
|
|
path_to_parent = opts.get_path_to_parent(model)
|
|
if path_to_parent:
|
|
path.extend(path_to_parent)
|
|
cur_names_with_path[1].extend(path_to_parent)
|
|
opts = path_to_parent[-1].to_opts
|
|
if hasattr(field, "path_infos"):
|
|
if filtered_relation:
|
|
pathinfos = field.get_path_info(filtered_relation)
|
|
else:
|
|
pathinfos = field.path_infos
|
|
if not allow_many:
|
|
for inner_pos, p in enumerate(pathinfos):
|
|
if p.m2m:
|
|
cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1])
|
|
names_with_path.append(cur_names_with_path)
|
|
raise MultiJoin(pos + 1, names_with_path)
|
|
last = pathinfos[-1]
|
|
path.extend(pathinfos)
|
|
final_field = last.join_field
|
|
opts = last.to_opts
|
|
targets = last.target_fields
|
|
cur_names_with_path[1].extend(pathinfos)
|
|
names_with_path.append(cur_names_with_path)
|
|
else:
|
|
# Local non-relational field.
|
|
final_field = field
|
|
targets = (field,)
|
|
if fail_on_missing and pos + 1 != len(names):
|
|
raise FieldError(
|
|
"Cannot resolve keyword %r into field. Join on '%s'"
|
|
" not permitted." % (names[pos + 1], name)
|
|
)
|
|
break
|
|
return path, final_field, targets, names[pos + 1 :]
|
|
|
|
def setup_joins(
|
|
self,
|
|
names,
|
|
opts,
|
|
alias,
|
|
can_reuse=None,
|
|
allow_many=True,
|
|
reuse_with_filtered_relation=False,
|
|
):
|
|
"""
|
|
Compute the necessary table joins for the passage through the fields
|
|
given in 'names'. 'opts' is the Options class for the current model
|
|
(which gives the table we are starting from), 'alias' is the alias for
|
|
the table to start the joining from.
|
|
|
|
The 'can_reuse' defines the reverse foreign key joins we can reuse. It
|
|
can be None in which case all joins are reusable or a set of aliases
|
|
that can be reused. Note that non-reverse foreign keys are always
|
|
reusable when using setup_joins().
|
|
|
|
The 'reuse_with_filtered_relation' can be used to force 'can_reuse'
|
|
parameter and force the relation on the given connections.
|
|
|
|
If 'allow_many' is False, then any reverse foreign key seen will
|
|
generate a MultiJoin exception.
|
|
|
|
Return the final field involved in the joins, the target field (used
|
|
for any 'where' constraint), the final 'opts' value, the joins, the
|
|
field path traveled to generate the joins, and a transform function
|
|
that takes a field and alias and is equivalent to `field.get_col(alias)`
|
|
in the simple case but wraps field transforms if they were included in
|
|
names.
|
|
|
|
The target field is the field containing the concrete value. Final
|
|
field can be something different, for example foreign key pointing to
|
|
that value. Final field is needed for example in some value
|
|
conversions (convert 'obj' in fk__id=obj to pk val using the foreign
|
|
key field for example).
|
|
"""
|
|
joins = [alias]
|
|
# The transform can't be applied yet, as joins must be trimmed later.
|
|
# To avoid making every caller of this method look up transforms
|
|
# directly, compute transforms here and create a partial that converts
|
|
# fields to the appropriate wrapped version.
|
|
|
|
def final_transformer(field, alias):
|
|
if not self.alias_cols:
|
|
alias = None
|
|
return field.get_col(alias)
|
|
|
|
# Try resolving all the names as fields first. If there's an error,
|
|
# treat trailing names as lookups until a field can be resolved.
|
|
last_field_exception = None
|
|
for pivot in range(len(names), 0, -1):
|
|
try:
|
|
path, final_field, targets, rest = self.names_to_path(
|
|
names[:pivot],
|
|
opts,
|
|
allow_many,
|
|
fail_on_missing=True,
|
|
)
|
|
except FieldError as exc:
|
|
if pivot == 1:
|
|
# The first item cannot be a lookup, so it's safe
|
|
# to raise the field error here.
|
|
raise
|
|
else:
|
|
last_field_exception = exc
|
|
else:
|
|
# The transforms are the remaining items that couldn't be
|
|
# resolved into fields.
|
|
transforms = names[pivot:]
|
|
break
|
|
for name in transforms:
|
|
|
|
def transform(field, alias, *, name, previous):
|
|
try:
|
|
wrapped = previous(field, alias)
|
|
return self.try_transform(wrapped, name)
|
|
except FieldError:
|
|
# FieldError is raised if the transform doesn't exist.
|
|
if isinstance(final_field, Field) and last_field_exception:
|
|
raise last_field_exception
|
|
else:
|
|
raise
|
|
|
|
final_transformer = functools.partial(
|
|
transform, name=name, previous=final_transformer
|
|
)
|
|
final_transformer.has_transforms = True
|
|
# Then, add the path to the query's joins. Note that we can't trim
|
|
# joins at this stage - we will need the information about join type
|
|
# of the trimmed joins.
|
|
for join in path:
|
|
if join.filtered_relation:
|
|
filtered_relation = join.filtered_relation.clone()
|
|
table_alias = filtered_relation.alias
|
|
else:
|
|
filtered_relation = None
|
|
table_alias = None
|
|
opts = join.to_opts
|
|
if join.direct:
|
|
nullable = self.is_nullable(join.join_field)
|
|
else:
|
|
nullable = True
|
|
connection = self.join_class(
|
|
opts.db_table,
|
|
alias,
|
|
table_alias,
|
|
INNER,
|
|
join.join_field,
|
|
nullable,
|
|
filtered_relation=filtered_relation,
|
|
)
|
|
reuse = can_reuse if join.m2m or reuse_with_filtered_relation else None
|
|
alias = self.join(
|
|
connection,
|
|
reuse=reuse,
|
|
reuse_with_filtered_relation=reuse_with_filtered_relation,
|
|
)
|
|
joins.append(alias)
|
|
if filtered_relation:
|
|
filtered_relation.path = joins[:]
|
|
return JoinInfo(final_field, targets, opts, joins, path, final_transformer)
|
|
|
|
def trim_joins(self, targets, joins, path):
|
|
"""
|
|
The 'target' parameter is the final field being joined to, 'joins'
|
|
is the full list of join aliases. The 'path' contain the PathInfos
|
|
used to create the joins.
|
|
|
|
Return the final target field and table alias and the new active
|
|
joins.
|
|
|
|
Always trim any direct join if the target column is already in the
|
|
previous table. Can't trim reverse joins as it's unknown if there's
|
|
anything on the other side of the join.
|
|
"""
|
|
joins = joins[:]
|
|
for pos, info in enumerate(reversed(path)):
|
|
if len(joins) == 1 or not info.direct:
|
|
break
|
|
if info.filtered_relation:
|
|
break
|
|
join_targets = {t.column for t in info.join_field.foreign_related_fields}
|
|
cur_targets = {t.column for t in targets}
|
|
if not cur_targets.issubset(join_targets):
|
|
break
|
|
targets_dict = {
|
|
r[1].column: r[0]
|
|
for r in info.join_field.related_fields
|
|
if r[1].column in cur_targets
|
|
}
|
|
targets = tuple(targets_dict[t.column] for t in targets)
|
|
self.unref_alias(joins.pop())
|
|
return targets, joins[-1], joins
|
|
|
|
@classmethod
|
|
def _gen_cols(cls, exprs, include_external=False, resolve_refs=True):
|
|
for expr in exprs:
|
|
if isinstance(expr, Col):
|
|
yield expr
|
|
elif include_external and callable(
|
|
getattr(expr, "get_external_cols", None)
|
|
):
|
|
yield from expr.get_external_cols()
|
|
elif hasattr(expr, "get_source_expressions"):
|
|
if not resolve_refs and isinstance(expr, Ref):
|
|
continue
|
|
yield from cls._gen_cols(
|
|
expr.get_source_expressions(),
|
|
include_external=include_external,
|
|
resolve_refs=resolve_refs,
|
|
)
|
|
|
|
@classmethod
|
|
def _gen_col_aliases(cls, exprs):
|
|
yield from (expr.alias for expr in cls._gen_cols(exprs))
|
|
|
|
def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False):
|
|
annotation = self.annotations.get(name)
|
|
if annotation is not None:
|
|
if not allow_joins:
|
|
for alias in self._gen_col_aliases([annotation]):
|
|
if isinstance(self.alias_map[alias], Join):
|
|
raise FieldError(
|
|
"Joined field references are not permitted in this query"
|
|
)
|
|
if summarize:
|
|
# Summarize currently means we are doing an aggregate() query
|
|
# which is executed as a wrapped subquery if any of the
|
|
# aggregate() elements reference an existing annotation. In
|
|
# that case we need to return a Ref to the subquery's annotation.
|
|
if name not in self.annotation_select:
|
|
raise FieldError(
|
|
"Cannot aggregate over the '%s' alias. Use annotate() "
|
|
"to promote it." % name
|
|
)
|
|
return Ref(name, self.annotation_select[name])
|
|
else:
|
|
return annotation
|
|
else:
|
|
field_list = name.split(LOOKUP_SEP)
|
|
annotation = self.annotations.get(field_list[0])
|
|
if annotation is not None:
|
|
for transform in field_list[1:]:
|
|
annotation = self.try_transform(annotation, transform)
|
|
return annotation
|
|
join_info = self.setup_joins(
|
|
field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse
|
|
)
|
|
targets, final_alias, join_list = self.trim_joins(
|
|
join_info.targets, join_info.joins, join_info.path
|
|
)
|
|
if not allow_joins and len(join_list) > 1:
|
|
raise FieldError(
|
|
"Joined field references are not permitted in this query"
|
|
)
|
|
if len(targets) > 1:
|
|
raise FieldError(
|
|
"Referencing multicolumn fields with F() objects isn't supported"
|
|
)
|
|
# Verify that the last lookup in name is a field or a transform:
|
|
# transform_function() raises FieldError if not.
|
|
transform = join_info.transform_function(targets[0], final_alias)
|
|
if reuse is not None:
|
|
reuse.update(join_list)
|
|
return transform
|
|
|
|
def split_exclude(self, filter_expr, can_reuse, names_with_path):
|
|
"""
|
|
When doing an exclude against any kind of N-to-many relation, we need
|
|
to use a subquery. This method constructs the nested query, given the
|
|
original exclude filter (filter_expr) and the portion up to the first
|
|
N-to-many relation field.
|
|
|
|
For example, if the origin filter is ~Q(child__name='foo'), filter_expr
|
|
is ('child__name', 'foo') and can_reuse is a set of joins usable for
|
|
filters in the original query.
|
|
|
|
We will turn this into equivalent of:
|
|
WHERE NOT EXISTS(
|
|
SELECT 1
|
|
FROM child
|
|
WHERE name = 'foo' AND child.parent_id = parent.id
|
|
LIMIT 1
|
|
)
|
|
"""
|
|
# Generate the inner query.
|
|
query = self.__class__(self.model)
|
|
query._filtered_relations = self._filtered_relations
|
|
filter_lhs, filter_rhs = filter_expr
|
|
if isinstance(filter_rhs, OuterRef):
|
|
filter_rhs = OuterRef(filter_rhs)
|
|
elif isinstance(filter_rhs, F):
|
|
filter_rhs = OuterRef(filter_rhs.name)
|
|
query.add_filter(filter_lhs, filter_rhs)
|
|
query.clear_ordering(force=True)
|
|
# Try to have as simple as possible subquery -> trim leading joins from
|
|
# the subquery.
|
|
trimmed_prefix, contains_louter = query.trim_start(names_with_path)
|
|
|
|
col = query.select[0]
|
|
select_field = col.target
|
|
alias = col.alias
|
|
if alias in can_reuse:
|
|
pk = select_field.model._meta.pk
|
|
# Need to add a restriction so that outer query's filters are in effect for
|
|
# the subquery, too.
|
|
query.bump_prefix(self)
|
|
lookup_class = select_field.get_lookup("exact")
|
|
# Note that the query.select[0].alias is different from alias
|
|
# due to bump_prefix above.
|
|
lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias))
|
|
query.where.add(lookup, AND)
|
|
query.external_aliases[alias] = True
|
|
|
|
lookup_class = select_field.get_lookup("exact")
|
|
lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix))
|
|
query.where.add(lookup, AND)
|
|
condition, needed_inner = self.build_filter(Exists(query))
|
|
|
|
if contains_louter:
|
|
or_null_condition, _ = self.build_filter(
|
|
("%s__isnull" % trimmed_prefix, True),
|
|
current_negated=True,
|
|
branch_negated=True,
|
|
can_reuse=can_reuse,
|
|
)
|
|
condition.add(or_null_condition, OR)
|
|
# Note that the end result will be:
|
|
# (outercol NOT IN innerq AND outercol IS NOT NULL) OR outercol IS NULL.
|
|
# This might look crazy but due to how IN works, this seems to be
|
|
# correct. If the IS NOT NULL check is removed then outercol NOT
|
|
# IN will return UNKNOWN. If the IS NULL check is removed, then if
|
|
# outercol IS NULL we will not match the row.
|
|
return condition, needed_inner
|
|
|
|
def set_empty(self):
|
|
self.where.add(NothingNode(), AND)
|
|
for query in self.combined_queries:
|
|
query.set_empty()
|
|
|
|
def is_empty(self):
|
|
return any(isinstance(c, NothingNode) for c in self.where.children)
|
|
|
|
def set_limits(self, low=None, high=None):
|
|
"""
|
|
Adjust the limits on the rows retrieved. Use low/high to set these,
|
|
as it makes it more Pythonic to read and write. When the SQL query is
|
|
created, convert them to the appropriate offset and limit values.
|
|
|
|
Apply any limits passed in here to the existing constraints. Add low
|
|
to the current low value and clamp both to any existing high value.
|
|
"""
|
|
if high is not None:
|
|
if self.high_mark is not None:
|
|
self.high_mark = min(self.high_mark, self.low_mark + high)
|
|
else:
|
|
self.high_mark = self.low_mark + high
|
|
if low is not None:
|
|
if self.high_mark is not None:
|
|
self.low_mark = min(self.high_mark, self.low_mark + low)
|
|
else:
|
|
self.low_mark = self.low_mark + low
|
|
|
|
if self.low_mark == self.high_mark:
|
|
self.set_empty()
|
|
|
|
def clear_limits(self):
|
|
"""Clear any existing limits."""
|
|
self.low_mark, self.high_mark = 0, None
|
|
|
|
@property
|
|
def is_sliced(self):
|
|
return self.low_mark != 0 or self.high_mark is not None
|
|
|
|
def has_limit_one(self):
|
|
return self.high_mark is not None and (self.high_mark - self.low_mark) == 1
|
|
|
|
def can_filter(self):
|
|
"""
|
|
Return True if adding filters to this instance is still possible.
|
|
|
|
Typically, this means no limits or offsets have been put on the results.
|
|
"""
|
|
return not self.is_sliced
|
|
|
|
def clear_select_clause(self):
|
|
"""Remove all fields from SELECT clause."""
|
|
self.select = ()
|
|
self.default_cols = False
|
|
self.select_related = False
|
|
self.set_extra_mask(())
|
|
self.set_annotation_mask(())
|
|
|
|
def clear_select_fields(self):
|
|
"""
|
|
Clear the list of fields to select (but not extra_select columns).
|
|
Some queryset types completely replace any existing list of select
|
|
columns.
|
|
"""
|
|
self.select = ()
|
|
self.values_select = ()
|
|
|
|
def add_select_col(self, col, name):
|
|
self.select += (col,)
|
|
self.values_select += (name,)
|
|
|
|
def set_select(self, cols):
|
|
self.default_cols = False
|
|
self.select = tuple(cols)
|
|
|
|
def add_distinct_fields(self, *field_names):
|
|
"""
|
|
Add and resolve the given fields to the query's "distinct on" clause.
|
|
"""
|
|
self.distinct_fields = field_names
|
|
self.distinct = True
|
|
|
|
def add_fields(self, field_names, allow_m2m=True):
|
|
"""
|
|
Add the given (model) fields to the select set. Add the field names in
|
|
the order specified.
|
|
"""
|
|
alias = self.get_initial_alias()
|
|
opts = self.get_meta()
|
|
|
|
try:
|
|
cols = []
|
|
for name in field_names:
|
|
# Join promotion note - we must not remove any rows here, so
|
|
# if there is no existing joins, use outer join.
|
|
join_info = self.setup_joins(
|
|
name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m
|
|
)
|
|
targets, final_alias, joins = self.trim_joins(
|
|
join_info.targets,
|
|
join_info.joins,
|
|
join_info.path,
|
|
)
|
|
for target in targets:
|
|
cols.append(join_info.transform_function(target, final_alias))
|
|
if cols:
|
|
self.set_select(cols)
|
|
except MultiJoin:
|
|
raise FieldError("Invalid field name: '%s'" % name)
|
|
except FieldError:
|
|
if LOOKUP_SEP in name:
|
|
# For lookups spanning over relationships, show the error
|
|
# from the model on which the lookup failed.
|
|
raise
|
|
elif name in self.annotations:
|
|
raise FieldError(
|
|
"Cannot select the '%s' alias. Use annotate() to promote "
|
|
"it." % name
|
|
)
|
|
else:
|
|
names = sorted(
|
|
[
|
|
*get_field_names_from_opts(opts),
|
|
*self.extra,
|
|
*self.annotation_select,
|
|
*self._filtered_relations,
|
|
]
|
|
)
|
|
raise FieldError(
|
|
"Cannot resolve keyword %r into field. "
|
|
"Choices are: %s" % (name, ", ".join(names))
|
|
)
|
|
|
|
def add_ordering(self, *ordering):
|
|
"""
|
|
Add items from the 'ordering' sequence to the query's "order by"
|
|
clause. These items are either field names (not column names) --
|
|
possibly with a direction prefix ('-' or '?') -- or OrderBy
|
|
expressions.
|
|
|
|
If 'ordering' is empty, clear all ordering from the query.
|
|
"""
|
|
errors = []
|
|
for item in ordering:
|
|
if isinstance(item, str):
|
|
if item == "?":
|
|
continue
|
|
if item.startswith("-"):
|
|
item = item[1:]
|
|
if item in self.annotations:
|
|
continue
|
|
if self.extra and item in self.extra:
|
|
continue
|
|
# names_to_path() validates the lookup. A descriptive
|
|
# FieldError will be raise if it's not.
|
|
self.names_to_path(item.split(LOOKUP_SEP), self.model._meta)
|
|
elif not hasattr(item, "resolve_expression"):
|
|
errors.append(item)
|
|
if getattr(item, "contains_aggregate", False):
|
|
raise FieldError(
|
|
"Using an aggregate in order_by() without also including "
|
|
"it in annotate() is not allowed: %s" % item
|
|
)
|
|
if errors:
|
|
raise FieldError("Invalid order_by arguments: %s" % errors)
|
|
if ordering:
|
|
self.order_by += ordering
|
|
else:
|
|
self.default_ordering = False
|
|
|
|
def clear_ordering(self, force=False, clear_default=True):
|
|
"""
|
|
Remove any ordering settings if the current query allows it without
|
|
side effects, set 'force' to True to clear the ordering regardless.
|
|
If 'clear_default' is True, there will be no ordering in the resulting
|
|
query (not even the model's default).
|
|
"""
|
|
if not force and (
|
|
self.is_sliced or self.distinct_fields or self.select_for_update
|
|
):
|
|
return
|
|
self.order_by = ()
|
|
self.extra_order_by = ()
|
|
if clear_default:
|
|
self.default_ordering = False
|
|
|
|
def set_group_by(self, allow_aliases=True):
|
|
"""
|
|
Expand the GROUP BY clause required by the query.
|
|
|
|
This will usually be the set of all non-aggregate fields in the
|
|
return data. If the database backend supports grouping by the
|
|
primary key, and the query would be equivalent, the optimization
|
|
will be made automatically.
|
|
"""
|
|
if allow_aliases and self.values_select:
|
|
# If grouping by aliases is allowed assign selected value aliases
|
|
# by moving them to annotations.
|
|
group_by_annotations = {}
|
|
values_select = {}
|
|
for alias, expr in zip(self.values_select, self.select):
|
|
if isinstance(expr, Col):
|
|
values_select[alias] = expr
|
|
else:
|
|
group_by_annotations[alias] = expr
|
|
self.annotations = {**group_by_annotations, **self.annotations}
|
|
self.append_annotation_mask(group_by_annotations)
|
|
self.select = tuple(values_select.values())
|
|
self.values_select = tuple(values_select)
|
|
group_by = list(self.select)
|
|
for alias, annotation in self.annotation_select.items():
|
|
if not (group_by_cols := annotation.get_group_by_cols()):
|
|
continue
|
|
if allow_aliases and not annotation.contains_aggregate:
|
|
group_by.append(Ref(alias, annotation))
|
|
else:
|
|
group_by.extend(group_by_cols)
|
|
self.group_by = tuple(group_by)
|
|
|
|
def add_select_related(self, fields):
|
|
"""
|
|
Set up the select_related data structure so that we only select
|
|
certain related models (as opposed to all models, when
|
|
self.select_related=True).
|
|
"""
|
|
if isinstance(self.select_related, bool):
|
|
field_dict = {}
|
|
else:
|
|
field_dict = self.select_related
|
|
for field in fields:
|
|
d = field_dict
|
|
for part in field.split(LOOKUP_SEP):
|
|
d = d.setdefault(part, {})
|
|
self.select_related = field_dict
|
|
|
|
def add_extra(self, select, select_params, where, params, tables, order_by):
|
|
"""
|
|
Add data to the various extra_* attributes for user-created additions
|
|
to the query.
|
|
"""
|
|
if select:
|
|
# We need to pair any placeholder markers in the 'select'
|
|
# dictionary with their parameters in 'select_params' so that
|
|
# subsequent updates to the select dictionary also adjust the
|
|
# parameters appropriately.
|
|
select_pairs = {}
|
|
if select_params:
|
|
param_iter = iter(select_params)
|
|
else:
|
|
param_iter = iter([])
|
|
for name, entry in select.items():
|
|
self.check_alias(name)
|
|
entry = str(entry)
|
|
entry_params = []
|
|
pos = entry.find("%s")
|
|
while pos != -1:
|
|
if pos == 0 or entry[pos - 1] != "%":
|
|
entry_params.append(next(param_iter))
|
|
pos = entry.find("%s", pos + 2)
|
|
select_pairs[name] = (entry, entry_params)
|
|
self.extra.update(select_pairs)
|
|
if where or params:
|
|
self.where.add(ExtraWhere(where, params), AND)
|
|
if tables:
|
|
self.extra_tables += tuple(tables)
|
|
if order_by:
|
|
self.extra_order_by = order_by
|
|
|
|
def clear_deferred_loading(self):
|
|
"""Remove any fields from the deferred loading set."""
|
|
self.deferred_loading = (frozenset(), True)
|
|
|
|
def add_deferred_loading(self, field_names):
|
|
"""
|
|
Add the given list of model field names to the set of fields to
|
|
exclude from loading from the database when automatic column selection
|
|
is done. Add the new field names to any existing field names that
|
|
are deferred (or removed from any existing field names that are marked
|
|
as the only ones for immediate loading).
|
|
"""
|
|
# Fields on related models are stored in the literal double-underscore
|
|
# format, so that we can use a set datastructure. We do the foo__bar
|
|
# splitting and handling when computing the SQL column names (as part of
|
|
# get_columns()).
|
|
existing, defer = self.deferred_loading
|
|
if defer:
|
|
# Add to existing deferred names.
|
|
self.deferred_loading = existing.union(field_names), True
|
|
else:
|
|
# Remove names from the set of any existing "immediate load" names.
|
|
if new_existing := existing.difference(field_names):
|
|
self.deferred_loading = new_existing, False
|
|
else:
|
|
self.clear_deferred_loading()
|
|
if new_only := set(field_names).difference(existing):
|
|
self.deferred_loading = new_only, True
|
|
|
|
def add_immediate_loading(self, field_names):
|
|
"""
|
|
Add the given list of model field names to the set of fields to
|
|
retrieve when the SQL is executed ("immediate loading" fields). The
|
|
field names replace any existing immediate loading field names. If
|
|
there are field names already specified for deferred loading, remove
|
|
those names from the new field_names before storing the new names
|
|
for immediate loading. (That is, immediate loading overrides any
|
|
existing immediate values, but respects existing deferrals.)
|
|
"""
|
|
existing, defer = self.deferred_loading
|
|
field_names = set(field_names)
|
|
if "pk" in field_names:
|
|
field_names.remove("pk")
|
|
field_names.add(self.get_meta().pk.name)
|
|
|
|
if defer:
|
|
# Remove any existing deferred names from the current set before
|
|
# setting the new names.
|
|
self.deferred_loading = field_names.difference(existing), False
|
|
else:
|
|
# Replace any existing "immediate load" field names.
|
|
self.deferred_loading = frozenset(field_names), False
|
|
|
|
def set_annotation_mask(self, names):
|
|
"""Set the mask of annotations that will be returned by the SELECT."""
|
|
if names is None:
|
|
self.annotation_select_mask = None
|
|
else:
|
|
self.annotation_select_mask = set(names)
|
|
self._annotation_select_cache = None
|
|
|
|
def append_annotation_mask(self, names):
|
|
if self.annotation_select_mask is not None:
|
|
self.set_annotation_mask(self.annotation_select_mask.union(names))
|
|
|
|
def set_extra_mask(self, names):
|
|
"""
|
|
Set the mask of extra select items that will be returned by SELECT.
|
|
Don't remove them from the Query since they might be used later.
|
|
"""
|
|
if names is None:
|
|
self.extra_select_mask = None
|
|
else:
|
|
self.extra_select_mask = set(names)
|
|
self._extra_select_cache = None
|
|
|
|
def set_values(self, fields):
|
|
self.select_related = False
|
|
self.clear_deferred_loading()
|
|
self.clear_select_fields()
|
|
self.has_select_fields = True
|
|
|
|
if fields:
|
|
field_names = []
|
|
extra_names = []
|
|
annotation_names = []
|
|
if not self.extra and not self.annotations:
|
|
# Shortcut - if there are no extra or annotations, then
|
|
# the values() clause must be just field names.
|
|
field_names = list(fields)
|
|
else:
|
|
self.default_cols = False
|
|
for f in fields:
|
|
if f in self.extra_select:
|
|
extra_names.append(f)
|
|
elif f in self.annotation_select:
|
|
annotation_names.append(f)
|
|
else:
|
|
field_names.append(f)
|
|
self.set_extra_mask(extra_names)
|
|
self.set_annotation_mask(annotation_names)
|
|
selected = frozenset(field_names + extra_names + annotation_names)
|
|
else:
|
|
field_names = [f.attname for f in self.model._meta.concrete_fields]
|
|
selected = frozenset(field_names)
|
|
# Selected annotations must be known before setting the GROUP BY
|
|
# clause.
|
|
if self.group_by is True:
|
|
self.add_fields(
|
|
(f.attname for f in self.model._meta.concrete_fields), False
|
|
)
|
|
# Disable GROUP BY aliases to avoid orphaning references to the
|
|
# SELECT clause which is about to be cleared.
|
|
self.set_group_by(allow_aliases=False)
|
|
self.clear_select_fields()
|
|
elif self.group_by:
|
|
# Resolve GROUP BY annotation references if they are not part of
|
|
# the selected fields anymore.
|
|
group_by = []
|
|
for expr in self.group_by:
|
|
if isinstance(expr, Ref) and expr.refs not in selected:
|
|
expr = self.annotations[expr.refs]
|
|
group_by.append(expr)
|
|
self.group_by = tuple(group_by)
|
|
|
|
self.values_select = tuple(field_names)
|
|
self.add_fields(field_names, True)
|
|
|
|
@property
|
|
def annotation_select(self):
|
|
"""
|
|
Return the dictionary of aggregate columns that are not masked and
|
|
should be used in the SELECT clause. Cache this result for performance.
|
|
"""
|
|
if self._annotation_select_cache is not None:
|
|
return self._annotation_select_cache
|
|
elif not self.annotations:
|
|
return {}
|
|
elif self.annotation_select_mask is not None:
|
|
self._annotation_select_cache = {
|
|
k: v
|
|
for k, v in self.annotations.items()
|
|
if k in self.annotation_select_mask
|
|
}
|
|
return self._annotation_select_cache
|
|
else:
|
|
return self.annotations
|
|
|
|
@property
|
|
def extra_select(self):
|
|
if self._extra_select_cache is not None:
|
|
return self._extra_select_cache
|
|
if not self.extra:
|
|
return {}
|
|
elif self.extra_select_mask is not None:
|
|
self._extra_select_cache = {
|
|
k: v for k, v in self.extra.items() if k in self.extra_select_mask
|
|
}
|
|
return self._extra_select_cache
|
|
else:
|
|
return self.extra
|
|
|
|
def trim_start(self, names_with_path):
|
|
"""
|
|
Trim joins from the start of the join path. The candidates for trim
|
|
are the PathInfos in names_with_path structure that are m2m joins.
|
|
|
|
Also set the select column so the start matches the join.
|
|
|
|
This method is meant to be used for generating the subquery joins &
|
|
cols in split_exclude().
|
|
|
|
Return a lookup usable for doing outerq.filter(lookup=self) and a
|
|
boolean indicating if the joins in the prefix contain a LEFT OUTER join.
|
|
_"""
|
|
all_paths = []
|
|
for _, paths in names_with_path:
|
|
all_paths.extend(paths)
|
|
contains_louter = False
|
|
# Trim and operate only on tables that were generated for
|
|
# the lookup part of the query. That is, avoid trimming
|
|
# joins generated for F() expressions.
|
|
lookup_tables = [
|
|
t for t in self.alias_map if t in self._lookup_joins or t == self.base_table
|
|
]
|
|
for trimmed_paths, path in enumerate(all_paths):
|
|
if path.m2m:
|
|
break
|
|
if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER:
|
|
contains_louter = True
|
|
alias = lookup_tables[trimmed_paths]
|
|
self.unref_alias(alias)
|
|
# The path.join_field is a Rel, lets get the other side's field
|
|
join_field = path.join_field.field
|
|
# Build the filter prefix.
|
|
paths_in_prefix = trimmed_paths
|
|
trimmed_prefix = []
|
|
for name, path in names_with_path:
|
|
if paths_in_prefix - len(path) < 0:
|
|
break
|
|
trimmed_prefix.append(name)
|
|
paths_in_prefix -= len(path)
|
|
trimmed_prefix.append(join_field.foreign_related_fields[0].name)
|
|
trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
|
|
# Lets still see if we can trim the first join from the inner query
|
|
# (that is, self). We can't do this for:
|
|
# - LEFT JOINs because we would miss those rows that have nothing on
|
|
# the outer side,
|
|
# - INNER JOINs from filtered relations because we would miss their
|
|
# filters.
|
|
first_join = self.alias_map[lookup_tables[trimmed_paths + 1]]
|
|
if first_join.join_type != LOUTER and not first_join.filtered_relation:
|
|
select_fields = [r[0] for r in join_field.related_fields]
|
|
select_alias = lookup_tables[trimmed_paths + 1]
|
|
self.unref_alias(lookup_tables[trimmed_paths])
|
|
extra_restriction = join_field.get_extra_restriction(
|
|
None, lookup_tables[trimmed_paths + 1]
|
|
)
|
|
if extra_restriction:
|
|
self.where.add(extra_restriction, AND)
|
|
else:
|
|
# TODO: It might be possible to trim more joins from the start of the
|
|
# inner query if it happens to have a longer join chain containing the
|
|
# values in select_fields. Lets punt this one for now.
|
|
select_fields = [r[1] for r in join_field.related_fields]
|
|
select_alias = lookup_tables[trimmed_paths]
|
|
# The found starting point is likely a join_class instead of a
|
|
# base_table_class reference. But the first entry in the query's FROM
|
|
# clause must not be a JOIN.
|
|
for table in self.alias_map:
|
|
if self.alias_refcount[table] > 0:
|
|
self.alias_map[table] = self.base_table_class(
|
|
self.alias_map[table].table_name,
|
|
table,
|
|
)
|
|
break
|
|
self.set_select([f.get_col(select_alias) for f in select_fields])
|
|
return trimmed_prefix, contains_louter
|
|
|
|
def is_nullable(self, field):
|
|
"""
|
|
Check if the given field should be treated as nullable.
|
|
|
|
Some backends treat '' as null and Django treats such fields as
|
|
nullable for those backends. In such situations field.null can be
|
|
False even if we should treat the field as nullable.
|
|
"""
|
|
# We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have
|
|
# (nor should it have) knowledge of which connection is going to be
|
|
# used. The proper fix would be to defer all decisions where
|
|
# is_nullable() is needed to the compiler stage, but that is not easy
|
|
# to do currently.
|
|
return field.null or (
|
|
field.empty_strings_allowed
|
|
and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
|
|
)
|
|
|
|
|
|
def get_order_dir(field, default="ASC"):
|
|
"""
|
|
Return the field name and direction for an order specification. For
|
|
example, '-foo' is returned as ('foo', 'DESC').
|
|
|
|
The 'default' param is used to indicate which way no prefix (or a '+'
|
|
prefix) should sort. The '-' prefix always sorts the opposite way.
|
|
"""
|
|
dirn = ORDER_DIR[default]
|
|
if field[0] == "-":
|
|
return field[1:], dirn[1]
|
|
return field, dirn[0]
|
|
|
|
|
|
class JoinPromoter:
|
|
"""
|
|
A class to abstract away join promotion problems for complex filter
|
|
conditions.
|
|
"""
|
|
|
|
def __init__(self, connector, num_children, negated):
|
|
self.connector = connector
|
|
self.negated = negated
|
|
if self.negated:
|
|
if connector == AND:
|
|
self.effective_connector = OR
|
|
else:
|
|
self.effective_connector = AND
|
|
else:
|
|
self.effective_connector = self.connector
|
|
self.num_children = num_children
|
|
# Maps of table alias to how many times it is seen as required for
|
|
# inner and/or outer joins.
|
|
self.votes = Counter()
|
|
|
|
def __repr__(self):
|
|
return (
|
|
f"{self.__class__.__qualname__}(connector={self.connector!r}, "
|
|
f"num_children={self.num_children!r}, negated={self.negated!r})"
|
|
)
|
|
|
|
def add_votes(self, votes):
|
|
"""
|
|
Add single vote per item to self.votes. Parameter can be any
|
|
iterable.
|
|
"""
|
|
self.votes.update(votes)
|
|
|
|
def update_join_types(self, query):
|
|
"""
|
|
Change join types so that the generated query is as efficient as
|
|
possible, but still correct. So, change as many joins as possible
|
|
to INNER, but don't make OUTER joins INNER if that could remove
|
|
results from the query.
|
|
"""
|
|
to_promote = set()
|
|
to_demote = set()
|
|
# The effective_connector is used so that NOT (a AND b) is treated
|
|
# similarly to (a OR b) for join promotion.
|
|
for table, votes in self.votes.items():
|
|
# We must use outer joins in OR case when the join isn't contained
|
|
# in all of the joins. Otherwise the INNER JOIN itself could remove
|
|
# valid results. Consider the case where a model with rel_a and
|
|
# rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now,
|
|
# if rel_a join doesn't produce any results is null (for example
|
|
# reverse foreign key or null value in direct foreign key), and
|
|
# there is a matching row in rel_b with col=2, then an INNER join
|
|
# to rel_a would remove a valid match from the query. So, we need
|
|
# to promote any existing INNER to LOUTER (it is possible this
|
|
# promotion in turn will be demoted later on).
|
|
if self.effective_connector == OR and votes < self.num_children:
|
|
to_promote.add(table)
|
|
# If connector is AND and there is a filter that can match only
|
|
# when there is a joinable row, then use INNER. For example, in
|
|
# rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL
|
|
# as join output, then the col=1 or col=2 can't match (as
|
|
# NULL=anything is always false).
|
|
# For the OR case, if all children voted for a join to be inner,
|
|
# then we can use INNER for the join. For example:
|
|
# (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell)
|
|
# then if rel_a doesn't produce any rows, the whole condition
|
|
# can't match. Hence we can safely use INNER join.
|
|
if self.effective_connector == AND or (
|
|
self.effective_connector == OR and votes == self.num_children
|
|
):
|
|
to_demote.add(table)
|
|
# Finally, what happens in cases where we have:
|
|
# (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0
|
|
# Now, we first generate the OR clause, and promote joins for it
|
|
# in the first if branch above. Both rel_a and rel_b are promoted
|
|
# to LOUTER joins. After that we do the AND case. The OR case
|
|
# voted no inner joins but the rel_a__col__gte=0 votes inner join
|
|
# for rel_a. We demote it back to INNER join (in AND case a single
|
|
# vote is enough). The demotion is OK, if rel_a doesn't produce
|
|
# rows, then the rel_a__col__gte=0 clause can't be true, and thus
|
|
# the whole clause must be false. So, it is safe to use INNER
|
|
# join.
|
|
# Note that in this example we could just as well have the __gte
|
|
# clause and the OR clause swapped. Or we could replace the __gte
|
|
# clause with an OR clause containing rel_a__col=1|rel_a__col=2,
|
|
# and again we could safely demote to INNER.
|
|
query.promote_joins(to_promote)
|
|
query.demote_joins(to_demote)
|
|
return to_demote
|