|
| 1 | +use ruff_macros::{ViolationMetadata, derive_message_formats}; |
| 2 | +use ruff_python_ast::name::QualifiedName; |
| 3 | +use ruff_python_ast::{self as ast, Expr, ExprCall, InterpolatedStringElement}; |
| 4 | +use ruff_python_semantic::{Modules, SemanticModel}; |
| 5 | +use ruff_text_size::Ranged; |
| 6 | + |
| 7 | +use crate::checkers::ast::Checker; |
| 8 | +use crate::rules::airflow::helpers::is_airflow_builtin_or_provider; |
| 9 | +use crate::{FixAvailability, Violation}; |
| 10 | + |
| 11 | +/// ## What it does |
| 12 | +/// Checks for calls to runtime-varying functions (such as `datetime.now()`) |
| 13 | +/// used as arguments in Airflow DAG or task constructors. |
| 14 | +/// |
| 15 | +/// ## Why is this bad? |
| 16 | +/// Using runtime-varying values as arguments to DAG or task constructors |
| 17 | +/// causes the serialized DAG hash to change on every parse, creating |
| 18 | +/// infinite DAG versions in the `dag_version` and `serialized_dag` tables. |
| 19 | +/// This leads to unbounded database growth and can eventually cause |
| 20 | +/// out-of-memory conditions. |
| 21 | +/// |
| 22 | +/// ## Example |
| 23 | +/// ```python |
| 24 | +/// from datetime import datetime |
| 25 | +/// |
| 26 | +/// from airflow import DAG |
| 27 | +/// |
| 28 | +/// dag = DAG(dag_id="my_dag", start_date=datetime.now()) |
| 29 | +/// ``` |
| 30 | +/// |
| 31 | +/// Use instead: |
| 32 | +/// ```python |
| 33 | +/// from datetime import datetime |
| 34 | +/// |
| 35 | +/// from airflow import DAG |
| 36 | +/// |
| 37 | +/// dag = DAG(dag_id="my_dag", start_date=datetime(2024, 1, 1)) |
| 38 | +/// ``` |
| 39 | +#[derive(ViolationMetadata)] |
| 40 | +#[violation_metadata(preview_since = "0.14.11")] |
| 41 | +pub(crate) struct Airflow3DagDynamicValue { |
| 42 | + function_name: String, |
| 43 | +} |
| 44 | + |
| 45 | +impl Violation for Airflow3DagDynamicValue { |
| 46 | + const FIX_AVAILABILITY: FixAvailability = FixAvailability::None; |
| 47 | + |
| 48 | + #[derive_message_formats] |
| 49 | + fn message(&self) -> String { |
| 50 | + let Airflow3DagDynamicValue { function_name } = self; |
| 51 | + format!( |
| 52 | + "`{function_name}()` produces a value that changes at runtime; using it in a DAG or task argument causes infinite DAG version creation" |
| 53 | + ) |
| 54 | + } |
| 55 | +} |
| 56 | + |
| 57 | +/// AIR304 |
| 58 | +pub(crate) fn airflow_3_dag_dynamic_value(checker: &Checker, call: &ExprCall) { |
| 59 | + if !checker.semantic().seen_module(Modules::AIRFLOW) { |
| 60 | + return; |
| 61 | + } |
| 62 | + |
| 63 | + let Some(qualified_name) = checker.semantic().resolve_qualified_name(&call.func) else { |
| 64 | + return; |
| 65 | + }; |
| 66 | + |
| 67 | + if !is_dag_or_task_constructor(&qualified_name) { |
| 68 | + return; |
| 69 | + } |
| 70 | + |
| 71 | + for keyword in &call.arguments.keywords { |
| 72 | + if let Some((expr, name)) = find_runtime_varying_call(&keyword.value, checker.semantic()) { |
| 73 | + checker.report_diagnostic( |
| 74 | + Airflow3DagDynamicValue { |
| 75 | + function_name: name.to_string(), |
| 76 | + }, |
| 77 | + expr.range(), |
| 78 | + ); |
| 79 | + } |
| 80 | + } |
| 81 | +} |
| 82 | + |
| 83 | +/// Check if the qualified name refers to a DAG constructor, `@dag` decorator, |
| 84 | +/// operator, sensor, or `@task` decorator. |
| 85 | +fn is_dag_or_task_constructor(qualified_name: &QualifiedName) -> bool { |
| 86 | + let segments = qualified_name.segments(); |
| 87 | + matches!(segments, ["airflow", .., "DAG" | "dag"]) |
| 88 | + || matches!(segments, ["airflow", "decorators" | "sdk", "task"]) |
| 89 | + || is_airflow_builtin_or_provider(segments, "operators", "Operator") |
| 90 | + || is_airflow_builtin_or_provider(segments, "sensors", "Sensor") |
| 91 | +} |
| 92 | + |
| 93 | +/// Recursively check an expression for calls to known runtime-varying functions. |
| 94 | +/// Returns the call expression and a display name (e.g., `"datetime.now"`) if found. |
| 95 | +fn find_runtime_varying_call<'a>( |
| 96 | + expr: &'a Expr, |
| 97 | + semantic: &SemanticModel, |
| 98 | +) -> Option<(&'a Expr, &'static str)> { |
| 99 | + match expr { |
| 100 | + Expr::Call(ExprCall { func, .. }) => { |
| 101 | + if let Some(qualified_name) = semantic.resolve_qualified_name(func) { |
| 102 | + let name = match qualified_name.segments() { |
| 103 | + ["datetime", "datetime", "now"] => Some("datetime.now"), |
| 104 | + ["datetime", "datetime", "utcnow"] => Some("datetime.utcnow"), |
| 105 | + ["datetime", "datetime", "today"] => Some("datetime.today"), |
| 106 | + ["datetime", "date", "today"] => Some("date.today"), |
| 107 | + ["pendulum", "now"] => Some("pendulum.now"), |
| 108 | + ["pendulum", "today"] => Some("pendulum.today"), |
| 109 | + ["pendulum", "yesterday"] => Some("pendulum.yesterday"), |
| 110 | + ["pendulum", "tomorrow"] => Some("pendulum.tomorrow"), |
| 111 | + ["time", "time"] => Some("time.time"), |
| 112 | + ["uuid", "uuid1"] => Some("uuid.uuid1"), |
| 113 | + ["uuid", "uuid4"] => Some("uuid.uuid4"), |
| 114 | + ["random", "random"] => Some("random.random"), |
| 115 | + ["random", "randint"] => Some("random.randint"), |
| 116 | + ["random", "choice"] => Some("random.choice"), |
| 117 | + ["random", "uniform"] => Some("random.uniform"), |
| 118 | + ["random", "randrange"] => Some("random.randrange"), |
| 119 | + ["random", "sample"] => Some("random.sample"), |
| 120 | + ["random", "getrandbits"] => Some("random.getrandbits"), |
| 121 | + _ => None, |
| 122 | + }; |
| 123 | + if let Some(name) = name { |
| 124 | + return Some((expr, name)); |
| 125 | + } |
| 126 | + } |
| 127 | + None |
| 128 | + } |
| 129 | + Expr::BinOp(ast::ExprBinOp { left, right, .. }) => { |
| 130 | + find_runtime_varying_call(left, semantic) |
| 131 | + .or_else(|| find_runtime_varying_call(right, semantic)) |
| 132 | + } |
| 133 | + Expr::UnaryOp(ast::ExprUnaryOp { operand, .. }) => { |
| 134 | + find_runtime_varying_call(operand, semantic) |
| 135 | + } |
| 136 | + Expr::Dict(ast::ExprDict { items, .. }) => items |
| 137 | + .iter() |
| 138 | + .find_map(|item| find_runtime_varying_call(&item.value, semantic)), |
| 139 | + Expr::List(ast::ExprList { elts, .. }) |
| 140 | + | Expr::Tuple(ast::ExprTuple { elts, .. }) |
| 141 | + | Expr::Set(ast::ExprSet { elts, .. }) => elts |
| 142 | + .iter() |
| 143 | + .find_map(|elt| find_runtime_varying_call(elt, semantic)), |
| 144 | + Expr::FString(ast::ExprFString { value, .. }) => value.elements().find_map(|element| { |
| 145 | + if let InterpolatedStringElement::Interpolation(interpolation) = element { |
| 146 | + find_runtime_varying_call(&interpolation.expression, semantic) |
| 147 | + } else { |
| 148 | + None |
| 149 | + } |
| 150 | + }), |
| 151 | + _ => None, |
| 152 | + } |
| 153 | +} |
0 commit comments