sqlglot.optimizer.canonicalize API documentation

Edit on GitHub

# sqlglot.optimizer.canonicalize

View Source

      1from __future__ import annotations
      2
      3import itertools
      4import typing as t
      5
      6from sqlglot import exp
      7from sqlglot.helper import is_date_unit, is_iso_date, is_iso_datetime
      8
      9
     10def canonicalize(expression: exp.Expression) -> exp.Expression:
     11    """Converts a sql expression into a standard form.
     12
     13    This method relies on annotate_types because many of the
     14    conversions rely on type inference.
     15
     16    Args:
     17        expression: The expression to canonicalize.
     18    """
     19    exp.replace_children(expression, canonicalize)
     20
     21    expression = add_text_to_concat(expression)
     22    expression = replace_date_funcs(expression)
     23    expression = coerce_type(expression)
     24    expression = remove_redundant_casts(expression)
     25    expression = ensure_bools(expression, _replace_int_predicate)
     26    expression = remove_ascending_order(expression)
     27
     28    return expression
     29
     30
     31def add_text_to_concat(node: exp.Expression) -> exp.Expression:
     32    if isinstance(node, exp.Add) and node.type and node.type.this in exp.DataType.TEXT_TYPES:
     33        node = exp.Concat(expressions=[node.left, node.right])
     34    return node
     35
     36
     37def replace_date_funcs(node: exp.Expression) -> exp.Expression:
     38    if isinstance(node, exp.Date) and not node.expressions and not node.args.get("zone"):
     39        return exp.cast(node.this, to=exp.DataType.Type.DATE)
     40    if isinstance(node, exp.Timestamp) and not node.expression:
     41        return exp.cast(node.this, to=exp.DataType.Type.TIMESTAMP)
     42    return node
     43
     44
     45COERCIBLE_DATE_OPS = (
     46    exp.Add,
     47    exp.Sub,
     48    exp.EQ,
     49    exp.NEQ,
     50    exp.GT,
     51    exp.GTE,
     52    exp.LT,
     53    exp.LTE,
     54    exp.NullSafeEQ,
     55    exp.NullSafeNEQ,
     56)
     57
     58
     59def coerce_type(node: exp.Expression) -> exp.Expression:
     60    if isinstance(node, COERCIBLE_DATE_OPS):
     61        _coerce_date(node.left, node.right)
     62    elif isinstance(node, exp.Between):
     63        _coerce_date(node.this, node.args["low"])
     64    elif isinstance(node, exp.Extract) and not node.expression.type.is_type(
     65        *exp.DataType.TEMPORAL_TYPES
     66    ):
     67        _replace_cast(node.expression, exp.DataType.Type.DATETIME)
     68    elif isinstance(node, (exp.DateAdd, exp.DateSub, exp.DateTrunc)):
     69        _coerce_timeunit_arg(node.this, node.unit)
     70    elif isinstance(node, exp.DateDiff):
     71        _coerce_datediff_args(node)
     72
     73    return node
     74
     75
     76def remove_redundant_casts(expression: exp.Expression) -> exp.Expression:
     77    if (
     78        isinstance(expression, exp.Cast)
     79        and expression.to.type
     80        and expression.this.type
     81        and expression.to.type.this == expression.this.type.this
     82    ):
     83        return expression.this
     84    return expression
     85
     86
     87def ensure_bools(
     88    expression: exp.Expression, replace_func: t.Callable[[exp.Expression], None]
     89) -> exp.Expression:
     90    if isinstance(expression, exp.Connector):
     91        replace_func(expression.left)
     92        replace_func(expression.right)
     93    elif isinstance(expression, exp.Not):
     94        replace_func(expression.this)
     95        # We can't replace num in CASE x WHEN num ..., because it's not the full predicate
     96    elif isinstance(expression, exp.If) and not (
     97        isinstance(expression.parent, exp.Case) and expression.parent.this
     98    ):
     99        replace_func(expression.this)
    100    elif isinstance(expression, (exp.Where, exp.Having)):
    101        replace_func(expression.this)
    102
    103    return expression
    104
    105
    106def remove_ascending_order(expression: exp.Expression) -> exp.Expression:
    107    if isinstance(expression, exp.Ordered) and expression.args.get("desc") is False:
    108        # Convert ORDER BY a ASC to ORDER BY a
    109        expression.set("desc", None)
    110
    111    return expression
    112
    113
    114def _coerce_date(a: exp.Expression, b: exp.Expression) -> None:
    115    for a, b in itertools.permutations([a, b]):
    116        if isinstance(b, exp.Interval):
    117            a = _coerce_timeunit_arg(a, b.unit)
    118        if (
    119            a.type
    120            and a.type.this == exp.DataType.Type.DATE
    121            and b.type
    122            and b.type.this
    123            not in (
    124                exp.DataType.Type.DATE,
    125                exp.DataType.Type.INTERVAL,
    126            )
    127        ):
    128            _replace_cast(b, exp.DataType.Type.DATE)
    129
    130
    131def _coerce_timeunit_arg(arg: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.Expression:
    132    if not arg.type:
    133        return arg
    134
    135    if arg.type.this in exp.DataType.TEXT_TYPES:
    136        date_text = arg.name
    137        is_iso_date_ = is_iso_date(date_text)
    138
    139        if is_iso_date_ and is_date_unit(unit):
    140            return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATE))
    141
    142        # An ISO date is also an ISO datetime, but not vice versa
    143        if is_iso_date_ or is_iso_datetime(date_text):
    144            return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATETIME))
    145
    146    elif arg.type.this == exp.DataType.Type.DATE and not is_date_unit(unit):
    147        return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATETIME))
    148
    149    return arg
    150
    151
    152def _coerce_datediff_args(node: exp.DateDiff) -> None:
    153    for e in (node.this, node.expression):
    154        if e.type.this not in exp.DataType.TEMPORAL_TYPES:
    155            e.replace(exp.cast(e.copy(), to=exp.DataType.Type.DATETIME))
    156
    157
    158def _replace_cast(node: exp.Expression, to: exp.DataType.Type) -> None:
    159    node.replace(exp.cast(node.copy(), to=to))
    160
    161
    162# this was originally designed for presto, there is a similar transform for tsql
    163# this is different in that it only operates on int types, this is because
    164# presto has a boolean type whereas tsql doesn't (people use bits)
    165# with y as (select true as x) select x = 0 FROM y -- illegal presto query
    166def _replace_int_predicate(expression: exp.Expression) -> None:
    167    if isinstance(expression, exp.Coalesce):
    168        for _, child in expression.iter_expressions():
    169            _replace_int_predicate(child)
    170    elif expression.type and expression.type.this in exp.DataType.INTEGER_TYPES:
    171        expression.replace(expression.neq(0))

def canonicalize( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    11def canonicalize(expression: exp.Expression) -> exp.Expression:
    12    """Converts a sql expression into a standard form.
    13
    14    This method relies on annotate_types because many of the
    15    conversions rely on type inference.
    16
    17    Args:
    18        expression: The expression to canonicalize.
    19    """
    20    exp.replace_children(expression, canonicalize)
    21
    22    expression = add_text_to_concat(expression)
    23    expression = replace_date_funcs(expression)
    24    expression = coerce_type(expression)
    25    expression = remove_redundant_casts(expression)
    26    expression = ensure_bools(expression, _replace_int_predicate)
    27    expression = remove_ascending_order(expression)
    28
    29    return expression

Converts a sql expression into a standard form.

This method relies on annotate_types because many of the conversions rely on type inference.

###### Arguments:

- expression: The expression to canonicalize.

def add_text_to_concat(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    32def add_text_to_concat(node: exp.Expression) -> exp.Expression:
    33    if isinstance(node, exp.Add) and node.type and node.type.this in exp.DataType.TEXT_TYPES:
    34        node = exp.Concat(expressions=[node.left, node.right])
    35    return node

def replace_date_funcs(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    38def replace_date_funcs(node: exp.Expression) -> exp.Expression:
    39    if isinstance(node, exp.Date) and not node.expressions and not node.args.get("zone"):
    40        return exp.cast(node.this, to=exp.DataType.Type.DATE)
    41    if isinstance(node, exp.Timestamp) and not node.expression:
    42        return exp.cast(node.this, to=exp.DataType.Type.TIMESTAMP)
    43    return node

COERCIBLE_DATE_OPS = (<class 'sqlglot.expressions.Add'>, <class 'sqlglot.expressions.Sub'>, <class 'sqlglot.expressions.EQ'>, <class 'sqlglot.expressions.NEQ'>, <class 'sqlglot.expressions.GT'>, <class 'sqlglot.expressions.GTE'>, <class 'sqlglot.expressions.LT'>, <class 'sqlglot.expressions.LTE'>, <class 'sqlglot.expressions.NullSafeEQ'>, <class 'sqlglot.expressions.NullSafeNEQ'>)
def coerce_type(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    60def coerce_type(node: exp.Expression) -> exp.Expression:
    61    if isinstance(node, COERCIBLE_DATE_OPS):
    62        _coerce_date(node.left, node.right)
    63    elif isinstance(node, exp.Between):
    64        _coerce_date(node.this, node.args["low"])
    65    elif isinstance(node, exp.Extract) and not node.expression.type.is_type(
    66        *exp.DataType.TEMPORAL_TYPES
    67    ):
    68        _replace_cast(node.expression, exp.DataType.Type.DATETIME)
    69    elif isinstance(node, (exp.DateAdd, exp.DateSub, exp.DateTrunc)):
    70        _coerce_timeunit_arg(node.this, node.unit)
    71    elif isinstance(node, exp.DateDiff):
    72        _coerce_datediff_args(node)
    73
    74    return node

def remove_redundant_casts( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    77def remove_redundant_casts(expression: exp.Expression) -> exp.Expression:
    78    if (
    79        isinstance(expression, exp.Cast)
    80        and expression.to.type
    81        and expression.this.type
    82        and expression.to.type.this == expression.this.type.this
    83    ):
    84        return expression.this
    85    return expression

def ensure_bools( expression: sqlglot.expressions.Expression, replace_func: Callable[[sqlglot.expressions.Expression], NoneType]) -> sqlglot.expressions.Expression: View Source

     88def ensure_bools(
     89    expression: exp.Expression, replace_func: t.Callable[[exp.Expression], None]
     90) -> exp.Expression:
     91    if isinstance(expression, exp.Connector):
     92        replace_func(expression.left)
     93        replace_func(expression.right)
     94    elif isinstance(expression, exp.Not):
     95        replace_func(expression.this)
     96        # We can't replace num in CASE x WHEN num ..., because it's not the full predicate
     97    elif isinstance(expression, exp.If) and not (
     98        isinstance(expression.parent, exp.Case) and expression.parent.this
     99    ):
    100        replace_func(expression.this)
    101    elif isinstance(expression, (exp.Where, exp.Having)):
    102        replace_func(expression.this)
    103
    104    return expression

def remove_ascending_order( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    107def remove_ascending_order(expression: exp.Expression) -> exp.Expression:
    108    if isinstance(expression, exp.Ordered) and expression.args.get("desc") is False:
    109        # Convert ORDER BY a ASC to ORDER BY a
    110        expression.set("desc", None)
    111
    112    return expression