Skip to content

Commit e4ccf88

Browse files
XianJiChenXianJiChen
authored
Add Terraform Support for Dataplex Discovery Scans (GoogleCloudPlatform#14274)
1 parent 16c6b8c commit e4ccf88

4 files changed

Lines changed: 203 additions & 3 deletions

File tree

‎mmv1/products/dataplex/Datascan.yaml‎

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,20 @@ examples:
9292
test_env_vars:
9393
project_name: 'PROJECT_NAME'
9494
exclude_docs: true
95+
- name: 'dataplex_datascan_basic_discovery'
96+
primary_resource_id: 'basic_discovery'
97+
vars:
98+
datascan_name: 'datadiscovery-basic'
99+
test_env_vars:
100+
project_name: 'PROJECT_NAME'
101+
location: 'REGION'
102+
- name: 'dataplex_datascan_full_discovery'
103+
primary_resource_id: 'full_discovery'
104+
vars:
105+
datascan_name: 'datadiscovery-full'
106+
test_env_vars:
107+
project_name: 'PROJECT_NAME'
108+
location: 'REGION'
95109
parameters:
96110
- name: 'location'
97111
type: String
@@ -170,7 +184,7 @@ properties:
170184
type: String
171185
description: |
172186
The service-qualified full resource name of the cloud resource for a DataScan job to scan against. The field could be:
173-
(Cloud Storage bucket for DataDiscoveryScan)BigQuery table of type "TABLE" for DataProfileScan/DataQualityScan.
187+
Cloud Storage bucket (//storage.googleapis.com/projects/PROJECT_ID/buckets/BUCKET_ID) for DataDiscoveryScan OR BigQuery table of type "TABLE" (/bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID) for DataProfileScan/DataQualityScan.
174188
immutable: true
175189
exactly_one_of:
176190
- 'data.0.entity'
@@ -242,13 +256,15 @@ properties:
242256
- 'DATA_SCAN_TYPE_UNSPECIFIED'
243257
- 'DATA_QUALITY'
244258
- 'DATA_PROFILE'
259+
- 'DATA_DISCOVERY'
245260
- name: 'dataQualitySpec'
246261
type: NestedObject
247262
description: |
248263
DataQualityScan related setting.
249264
exactly_one_of:
250265
- 'data_quality_spec'
251266
- 'data_profile_spec'
267+
- 'data_discovery_spec'
252268
properties:
253269
- name: 'samplingPercent'
254270
type: Double
@@ -498,6 +514,7 @@ properties:
498514
exactly_one_of:
499515
- 'data_quality_spec'
500516
- 'data_profile_spec'
517+
- 'data_discovery_spec'
501518
properties:
502519
- name: 'samplingPercent'
503520
type: Double
@@ -552,3 +569,94 @@ properties:
552569
For instance, if 'x' is of nested field type, listing 'x' is supported but 'x.y.z' is not supported. Here 'y' and 'y.z' are nested fields of 'x'.
553570
item_type:
554571
type: String
572+
- name: 'dataDiscoverySpec'
573+
type: NestedObject
574+
description: |
575+
DataDiscoveryScan related setting.
576+
send_empty_value: true
577+
allow_empty_object: true
578+
exactly_one_of:
579+
- 'data_quality_spec'
580+
- 'data_profile_spec'
581+
- 'data_discovery_spec'
582+
properties:
583+
- name: 'bigqueryPublishingConfig'
584+
type: NestedObject
585+
description: |
586+
Configuration for metadata publishing.
587+
properties:
588+
- name: tableType
589+
type: Enum
590+
description: |
591+
Determines whether to publish discovered tables as BigLake external tables or non-BigLake external tables.
592+
enum_values:
593+
- TABLE_TYPE_UNSPECIFIED
594+
- EXTERNAL
595+
- BIGLAKE
596+
- name: connection
597+
type: String
598+
description: |
599+
The BigQuery connection used to create BigLake tables. Must be in the form `projects/{projectId}/locations/{locationId}/connections/{connection_id}`.
600+
- name: location
601+
type: String
602+
description: |
603+
The location of the BigQuery dataset to publish BigLake external or non-BigLake external tables to.
604+
- name: project
605+
type: String
606+
description: |
607+
The project of the BigQuery dataset to publish BigLake external or non-BigLake external tables to. If not specified, the project of the Cloud Storage bucket will be used. The format is "projects/{project_id_or_number}".
608+
- name: 'storageConfig'
609+
type: NestedObject
610+
description: |
611+
Configurations related to Cloud Storage as the data source.
612+
properties:
613+
- name: includePatterns
614+
type: Array
615+
description: |
616+
Defines the data to include during discovery when only a subset of the data should be considered. Provide a list of patterns that identify the data to include. For Cloud Storage bucket assets, these patterns are interpreted as glob patterns used to match object names. For BigQuery dataset assets, these patterns are interpreted as patterns to match table names.
617+
item_type:
618+
type: String
619+
- name: excludePatterns
620+
type: Array
621+
description: |
622+
Defines the data to exclude during discovery. Provide a list of patterns that identify the data to exclude. For Cloud Storage bucket assets, these patterns are interpreted as glob patterns used to match object names. For BigQuery dataset assets, these patterns are interpreted as patterns to match table names.
623+
item_type:
624+
type: String
625+
- name: csvOptions
626+
type: NestedObject
627+
description: |
628+
Configuration for CSV data.
629+
properties:
630+
- name: headerRows
631+
type: Integer
632+
description: |
633+
The number of rows to interpret as header rows that should be skipped when reading data rows.
634+
- name: delimiter
635+
type: String
636+
description: |
637+
The delimiter that is used to separate values. The default is `,` (comma).
638+
- name: encoding
639+
type: String
640+
description: |
641+
The character encoding of the data. The default is UTF-8.
642+
- name: typeInferenceDisabled
643+
type: Boolean
644+
description: |
645+
Whether to disable the inference of data types for CSV data. If true, all columns are registered as strings.
646+
- name: quote
647+
type: String
648+
description: |
649+
The character used to quote column values. Accepts `"` (double quotation mark) or `'` (single quotation mark). If unspecified, defaults to `"` (double quotation mark).
650+
- name: jsonOptions
651+
type: NestedObject
652+
description: |
653+
Configuration for JSON data.
654+
properties:
655+
- name: encoding
656+
type: String
657+
description: |
658+
The character encoding of the data. The default is UTF-8.
659+
- name: typeInferenceDisabled
660+
type: Boolean
661+
description: |
662+
Whether to disable the inference of data types for JSON data. If true, all columns are registered as their primitive types (strings, number, or boolean).

‎mmv1/templates/terraform/examples/dataplex_datascan_basic_discovery.tf.tmpl‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
2+
location = "us-central1"
3+
data_scan_id = "{{index $.Vars "datascan_name"}}"
4+
5+
data {
6+
resource = "//storage.googleapis.com/projects/${google_storage_bucket.tf_test_bucket.project}/buckets/${google_storage_bucket.tf_test_bucket.name}"
7+
}
8+
9+
execution_spec {
10+
trigger {
11+
on_demand {}
12+
}
13+
}
14+
15+
data_discovery_spec {}
16+
17+
project = "{{index $.TestEnvVars "project_name"}}"
18+
}
19+
20+
resource "google_storage_bucket" "tf_test_bucket" {
21+
name = "tf-test-bucket-name-%{random_suffix}"
22+
location = "{{index $.TestEnvVars "location"}}"
23+
uniform_bucket_level_access = true
24+
}

‎mmv1/templates/terraform/examples/dataplex_datascan_full_discovery.tf.tmpl‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
resource "google_dataplex_datascan" "{{$.PrimaryResourceId}}" {
2+
location = "us-central1"
3+
display_name = "Full Datascan Discovery"
4+
data_scan_id = "{{index $.Vars "datascan_name"}}"
5+
description = "Example resource - Full Datascan Discovery"
6+
labels = {
7+
author = "billing"
8+
}
9+
10+
data {
11+
resource = "//storage.googleapis.com/projects/${google_storage_bucket.tf_test_bucket.project}/buckets/${google_storage_bucket.tf_test_bucket.name}"
12+
}
13+
14+
execution_spec {
15+
trigger {
16+
schedule {
17+
cron = "TZ=America/New_York 1 1 * * *"
18+
}
19+
}
20+
}
21+
22+
data_discovery_spec {
23+
bigquery_publishing_config {
24+
table_type = "BIGLAKE"
25+
connection = "projects/${google_bigquery_connection.tf_test_connection.project}/locations/${google_bigquery_connection.tf_test_connection.location}/connections/${google_bigquery_connection.tf_test_connection.connection_id}"
26+
location = "${google_storage_bucket.tf_test_bucket.location}"
27+
project = "projects/${google_storage_bucket.tf_test_bucket.project}"
28+
}
29+
30+
storage_config {
31+
include_patterns = [
32+
"ai*",
33+
"ml*",
34+
]
35+
exclude_patterns = [
36+
"doc*",
37+
"gen*",
38+
]
39+
csv_options {
40+
header_rows = 5
41+
delimiter = ","
42+
encoding = "UTF-8"
43+
type_inference_disabled = false
44+
quote = "'"
45+
}
46+
json_options {
47+
encoding = "UTF-8"
48+
type_inference_disabled = false
49+
}
50+
}
51+
}
52+
53+
project = "{{index $.TestEnvVars "project_name"}}"
54+
}
55+
56+
resource "google_storage_bucket" "tf_test_bucket" {
57+
name = "tf-test-bucket-name-%{random_suffix}"
58+
location = "{{index $.TestEnvVars "location"}}"
59+
uniform_bucket_level_access = true
60+
}
61+
62+
resource "google_bigquery_connection" "tf_test_connection" {
63+
connection_id = "tf-test-connection-%{random_suffix}"
64+
location = "us-central1"
65+
friendly_name = "tf-test-connection-%{random_suffix}"
66+
description = "a bigquery connection for tf test"
67+
cloud_resource {}
68+
}

‎mmv1/templates/terraform/examples/dataplex_datascan_full_quality_test.tf.tmpl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
resource "google_bigquery_dataset" "tf_test_dataset" {
2-
dataset_id = "tf_test_dataset_id"
2+
dataset_id = "tf_test_dataset_id_%{random_suffix}"
33
default_table_expiration_ms = 3600000
44
}
55

66
resource "google_bigquery_table" "tf_test_table" {
77
dataset_id = google_bigquery_dataset.tf_test_dataset.dataset_id
8-
table_id = "tf_test_table_id"
8+
table_id = "tf_test_table_id_%{random_suffix}"
99
deletion_protection = false
1010
schema = <<EOF
1111
[

0 commit comments

Comments
 (0)