The problem I’m having
Trying to set variables to use the same dates in both the main model and a macro called via pre hook, but the dates are not matching in the result despite using the same variables.
I think dbt runs the code twice, once to create the dag, then to run the model. It seems like the result of the first pass is remaining in place somehow for the macro. I’m trying to understand how to control for this.
Some example code or error messages
Model code:
{% set default_start = (modules.datetime.datetime.now() - modules.datetime.timedelta(days=6)) %}
{% set default_end = modules.datetime.datetime.now() %}
{% set run_start = (run_started_at - modules.datetime.timedelta(days=6)).astimezone(modules.pytz.timezone("America/Los_Angeles")) %}
{% set run_end = run_started_at.astimezone(modules.pytz.timezone("America/Los_Angeles")) %}
{% set pre_hook_text = '{{ test_time_macro("' ~ default_start ~ '", "' ~ default_end ~ '", "' ~ run_start ~ '", "' ~ run_end ~ '") }}'%}
{{ log(pre_hook_text, info=True) }}
{{ log("main sql default_start " ~ default_start, info=True) }}
{{ log("main sql default_end " ~ default_end, info=True) }}
{{ log("main sql run_start " ~ run_start, info=True) }}
{{ log("main sql run_end " ~ run_end, info=True) }}
{{
config(
materialized = 'incremental',
full_refresh = false,
pre_hook = before_begin(pre_hook_text)
)
}}
select 'MAINSQL' as xxx, '{{ default_start }}' as def_start, '{{ default_end }}' as def_end, '{{ run_start }}' as run_start, '{{ run_end }}' as run_end
{% if not is_incremental() %}
WHERE 1 = 2
{% endif %}
Macro code:
{% macro test_time_macro(d_start, d_end, r_start, r_end) %}
{%- if execute %}
{%- set source_relation = adapter.get_relation(
database=model.database,
schema=model.schema,
identifier=model.name) -%}
{% set table_exists=source_relation is not none %}
{{ log("macro sql default_start " ~ d_start, info=True) }}
{{ log("macro sql default_end " ~ d_end, info=True) }}
{{ log("macro sql run_start " ~ r_start, info=True) }}
{{ log("macro sql run_end " ~ r_end, info=True) }}
{% if table_exists %}
{{ log("table exists...running", info=True) }}
{% set delete_query %}
insert into {{ this }}
select 'MACRSQL' as xxx, '{{ d_start }}' as def_start, '{{ d_end }}' as def_end, '{{ r_start }}' as run_start, '{{ r_end }}' as run_end
{% endset %}
{% do run_query(delete_query) %}
{% else %}
{{ log("Table does not exist...will not run", info=True) }}
{% endif %}
{% endif %}
{% endmacro %}