-
Notifications
You must be signed in to change notification settings - Fork 202
Expand file tree
/
Copy pathMakefile
More file actions
173 lines (135 loc) · 6.96 KB
/
Copy pathMakefile
File metadata and controls
173 lines (135 loc) · 6.96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
#
# Local Env/Dev Setup
#
# Controls the number of parallel workers running tests. Try "make -e PARALLELISM=4 test".
PARALLELISM = auto
# Additional command line options to pass to pytest.
ADDITIONAL_PYTEST_OPTIONS =
# Additional command line options to pass to `pre-commit`
ADDITIONAL_PRECOMMIT_OPTIONS =
PERFORMANCE_OUTPUT_FILE = performance-report.json
PERFORMANCE_COMPARISON_OUTPUT_FILE = performance-comparison.md
TESTS_PERFORMANCE = tests_metricflow/performance
# Pytest that can populate the persistent source schema
USE_PERSISTENT_SOURCE_SCHEMA = --use-persistent-source-schema
TESTS_DBT_METRICFLOW = tests_dbt_metricflow
TESTS_METRICFLOW = tests_metricflow
TESTS_METRICFLOW_SEMANTICS = tests_metricflow_semantics
TESTS_METRICFLOW_SEMANTIC_INTERFACES = tests_metricflow_semantic_interfaces
POPULATE_PERSISTENT_SOURCE_SCHEMA = $(TESTS_METRICFLOW)/source_schema_tools.py::populate_source_schema
# Install Hatch package / project manager
.PHONY: install-hatch
install-hatch:
pip3 install hatch
.PHONY: perf
perf:
hatch -v run dev-env:pytest -vv -n 1 $(ADDITIONAL_PYTEST_OPTIONS) --output-json $(PERFORMANCE_OUTPUT_FILE) $(TESTS_PERFORMANCE)/
.PHONY: perf-compare
perf-compare:
hatch -v run dev-env:python $(TESTS_PERFORMANCE)/compare_reports.py $A $B $(PERFORMANCE_COMPARISON_OUTPUT_FILE)
# Testing and linting
.PHONY: test
test:
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW_SEMANTICS)/
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW_SEMANTIC_INTERFACES)/
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) -m "not slow" $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: test-include-slow-metricflow
test-include-slow-metricflow:
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW_SEMANTICS)/
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW_SEMANTIC_INTERFACES)/
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: test-include-slow-dbt-metricflow
test-include-slow-dbt-metricflow:
cd dbt-metricflow && hatch -v run dev-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_DBT_METRICFLOW)/
.PHONY: test-include-slow
test-include-slow: test-include-slow-metricflow test-include-slow-dbt-metricflow
.PHONY: test-postgres
test-postgres:
hatch -v run postgres-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
# Engine-specific test environments.
.PHONY: test-bigquery
test-bigquery:
hatch -v run bigquery-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: populate-persistent-source-schema-bigquery
populate-persistent-source-schema-bigquery:
hatch -v run bigquery-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
.PHONY: test-databricks
test-databricks:
hatch -v run databricks-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: populate-persistent-source-schema-databricks
populate-persistent-source-schema-databricks:
hatch -v run databricks-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
.PHONY: test-redshift
test-redshift:
hatch -v run redshift-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: populate-persistent-source-schema-redshift
populate-persistent-source-schema-redshift:
hatch -v run redshift-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
.PHONY: test-snowflake
test-snowflake:
hatch -v run snowflake-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: populate-persistent-source-schema-snowflake
populate-persistent-source-schema-snowflake:
hatch -v run snowflake-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
.PHONY: test-trino
test-trino:
hatch -v run trino-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: test-vertica
test-vertica:
hatch -v run vertica-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: test-athena
test-athena:
hatch -v run athena-env:pytest -vv -n $(PARALLELISM) $(ADDITIONAL_PYTEST_OPTIONS) $(TESTS_METRICFLOW)/
.PHONY: populate-persistent-source-schema-athena
populate-persistent-source-schema-athena:
hatch -v run athena-env:pytest -vv $(ADDITIONAL_PYTEST_OPTIONS) $(USE_PERSISTENT_SOURCE_SCHEMA) $(POPULATE_PERSISTENT_SOURCE_SCHEMA)
.PHONY: lint
lint:
hatch -v run dev-env:pre-commit run --verbose --all-files $(ADDITIONAL_PRECOMMIT_OPTIONS)
cd dbt-metricflow && hatch -v run dev-env:pre-commit run --verbose --all-files $(ADDITIONAL_PRECOMMIT_OPTIONS)
# Running data warehouses locally
.PHONY: postgres
postgres:
$(MAKE) -C local-data-warehouses postgres
.PHONY: trino
trino:
$(MAKE) -C local-data-warehouses trino
.PHONY: vertica
vertica:
$(MAKE) -C local-data-warehouses vertica
# Re-generate test snapshots using all supported SQL engines, or one engine when ENGINE is set.
.PHONY: regenerate-test-snapshots
regenerate-test-snapshots:
python3 -m scripts.generate_snapshots $(if $(ENGINE),--engine=$(ENGINE))
# Populate persistent source schemas for all relevant SQL engines, or one engine when ENGINE is set.
.PHONY: populate-persistent-source-schemas
populate-persistent-source-schemas:
python3 -m scripts.populate_persistent_source_schemas $(if $(ENGINE),--engine=$(ENGINE))
# Sync dbt-semantic-interfaces files to metricflow-semantic-interfaces folder
.PHONY: sync-dsi
sync-dsi:
python scripts/sync_dsi.py
# Re-generate snapshots for the default SQL engine.
.PHONY: test-snap
test-snap:
$(MAKE) test ADDITIONAL_PYTEST_OPTIONS=--overwrite-snapshots
.PHONY: testx
testx:
$(MAKE) test ADDITIONAL_PYTEST_OPTIONS=-x
.PHONY: testx-snap
testx-snap:
$(MAKE) test ADDITIONAL_PYTEST_OPTIONS='-x --overwrite-snapshots'
.PHONY: test-snap-slow
test-snap-slow:
cd dbt-metricflow && hatch -v run dev-env:pytest -vv -n $(PARALLELISM) --overwrite-snapshots $(TESTS_DBT_METRICFLOW)/
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) --overwrite-snapshots $(TESTS_METRICFLOW_SEMANTICS)/
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) --overwrite-snapshots $(TESTS_METRICFLOW_SEMANTIC_INTERFACES)/
hatch -v run dev-env:pytest -vv -n $(PARALLELISM) --overwrite-snapshots $(TESTS_METRICFLOW)/
.PHONY: test-build-packages-metricflow
test-build-packages-metricflow:
PYTHONPATH=. python scripts/ci_tests/run_package_build_tests.py --metricflow-repo-directory=. --package metricflow
.PHONY: test-build-packages-dbt-metricflow
test-build-packages-dbt-metricflow:
PYTHONPATH=. python scripts/ci_tests/run_package_build_tests.py --metricflow-repo-directory=. --package dbt-metricflow
.PHONY: test-build-packages
test-build-packages: test-build-packages-metricflow test-build-packages-dbt-metricflow