Skip to content

feat(bigquery): support managed iceberg tables #11931

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 31, 2025
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions bigquery/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,9 @@
// where 12345 is parent id. The value is the friendly short name of the
// tag value, e.g. "production".
ResourceTags map[string]string

// Specifies the configuration of a BigQuery table for Apache Iceberg (formerly BigLake Managed Table).
BigLakeConfiguration *BigLakeConfiguration
}

// TableConstraints defines the primary key and foreign key of a table.
Expand Down Expand Up @@ -380,6 +383,71 @@
}
}

// Represents the file format for Managed Tables for Apache Iceberg.

Check failure on line 386 in bigquery/table.go

View workflow job for this annotation

GitHub Actions / vet

comment on exported type BigLakeFileFormat should be of the form "BigLakeFileFormat ..." (with optional leading article)
type BigLakeFileFormat string

var (
// Default value.

Check failure on line 390 in bigquery/table.go

View workflow job for this annotation

GitHub Actions / vet

comment on exported var UnspecifiedBigLakeFileFormat should be of the form "UnspecifiedBigLakeFileFormat ..."
UnspecifiedBigLakeFileFormat BigLakeFileFormat = "FILE_FORMAT_UNSPECIFIED"
// Apache Parquet Format.

Check failure on line 392 in bigquery/table.go

View workflow job for this annotation

GitHub Actions / vet

comment on exported var ParquetBigLakeFileFormat should be of the form "ParquetBigLakeFileFormat ..."
ParquetBigLakeFileFormat BigLakeFileFormat = "PARQUET"
)

// Represents the table metadata format for Managed Tables for Apache Iceberg.

Check failure on line 396 in bigquery/table.go

View workflow job for this annotation

GitHub Actions / vet

comment on exported type BigLakeTableFormat should be of the form "BigLakeTableFormat ..." (with optional leading article)
type BigLakeTableFormat string

var (
// Default value.

Check failure on line 400 in bigquery/table.go

View workflow job for this annotation

GitHub Actions / vet

comment on exported var UnspecifiedBigLakeTableFormat should be of the form "UnspecifiedBigLakeTableFormat ..."
UnspecifiedBigLakeTableFormat BigLakeTableFormat = "TABLE_FORMAT_UNSPECIFIED"
// Apache Iceberg Format.

Check failure on line 402 in bigquery/table.go

View workflow job for this annotation

GitHub Actions / vet

comment on exported var IcebergBigLakeTableFormat should be of the form "IcebergBigLakeTableFormat ..."
IcebergBigLakeTableFormat BigLakeTableFormat = "ICEBERG"
)

// BigLakeConfiguration is used to configure aspects of BigQuery tables for
// Apache Iceberg (previously known as BigLake managed tables).
type BigLakeConfiguration struct {
// Optional. The connection specifying the credentials to be used to read and
// write to external storage, such as Cloud Storage. The connection_id can
// have the form `{project}.{location}.{connection_id}` or
// `projects/{project}/locations/{location}/connections/{connection_id}".
ConnectionID string

// Optional. The fully qualified location prefix of the external folder where
// table data is stored. The '*' wildcard character is not allowed. The URI
// should be in the format `gs://bucket/path_to_table/`
StorageURI string

// Optional. The file format the table data is stored in.
FileFormat BigLakeFileFormat

// Optional. The table format the metadata only snapshots are stored in.
TableFormat BigLakeTableFormat
}

func (blc *BigLakeConfiguration) toBQ() *bq.BigLakeConfiguration {
if blc == nil {
return nil
}
return &bq.BigLakeConfiguration{
ConnectionId: blc.ConnectionID,
StorageUri: blc.StorageURI,
FileFormat: string(blc.FileFormat),
TableFormat: string(blc.TableFormat),
}
}

func bqToBigLakeConfiguration(in *bq.BigLakeConfiguration) *BigLakeConfiguration {
if in == nil {
return nil
}
return &BigLakeConfiguration{
ConnectionID: in.ConnectionId,
StorageURI: in.StorageUri,
FileFormat: BigLakeFileFormat(in.FileFormat),
TableFormat: BigLakeTableFormat(in.TableFormat),
}
}

// SnapshotDefinition provides metadata related to the origin of a snapshot.
type SnapshotDefinition struct {

Expand Down Expand Up @@ -769,6 +837,7 @@
t.RequirePartitionFilter = tm.RequirePartitionFilter
t.SnapshotDefinition = tm.SnapshotDefinition.toBQ()
t.CloneDefinition = tm.CloneDefinition.toBQ()
t.BiglakeConfiguration = tm.BigLakeConfiguration.toBQ()

if !validExpiration(tm.ExpirationTime) {
return nil, fmt.Errorf("invalid expiration time: %v.\n"+
Expand Down Expand Up @@ -917,6 +986,7 @@
RequirePartitionFilter: t.RequirePartitionFilter,
SnapshotDefinition: bqToSnapshotDefinition(t.SnapshotDefinition, c),
CloneDefinition: bqToCloneDefinition(t.CloneDefinition, c),
BigLakeConfiguration: bqToBigLakeConfiguration(t.BiglakeConfiguration),
}
if t.MaterializedView != nil {
md.MaterializedView = bqToMaterializedViewDefinition(t.MaterializedView)
Expand Down Expand Up @@ -1145,6 +1215,10 @@
}
forceSend("ResourceTags")
}
if tm.BigLakeConfiguration != nil {
t.BiglakeConfiguration = tm.BigLakeConfiguration.toBQ()
forceSend("BigLakeConfiguration")
}
labels, forces, nulls := tm.update()
t.Labels = labels
t.ForceSendFields = append(t.ForceSendFields, forces...)
Expand Down Expand Up @@ -1239,6 +1313,9 @@
// tag value, e.g. "production".
ResourceTags map[string]string

// Update the configuration of a BigQuery table for Apache Iceberg (formerly BigLake Managed Table).
BigLakeConfiguration *BigLakeConfiguration

labelUpdater
}

Expand Down
24 changes: 24 additions & 0 deletions bigquery/table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ func TestBQToTableMetadata(t *testing.T) {
"key1": "val1",
"key2": "val2",
},
BiglakeConfiguration: &bq.BigLakeConfiguration{
ConnectionId: "bigconn",
StorageUri: "biguri",
FileFormat: "PARQUET",
TableFormat: "ICEBERG",
},
},
&TableMetadata{
Description: "desc",
Expand Down Expand Up @@ -168,6 +174,12 @@ func TestBQToTableMetadata(t *testing.T) {
"key1": "val1",
"key2": "val2",
},
BigLakeConfiguration: &BigLakeConfiguration{
ConnectionID: "bigconn",
StorageURI: "biguri",
FileFormat: ParquetBigLakeFileFormat,
TableFormat: IcebergBigLakeTableFormat,
},
},
},
} {
Expand Down Expand Up @@ -208,6 +220,12 @@ func TestTableMetadataToBQ(t *testing.T) {
"key1": "val1",
"key2": "val2",
},
BigLakeConfiguration: &BigLakeConfiguration{
ConnectionID: "bigconn",
StorageURI: "biguri",
FileFormat: ParquetBigLakeFileFormat,
TableFormat: IcebergBigLakeTableFormat,
},
},
&bq.Table{
FriendlyName: "n",
Expand All @@ -228,6 +246,12 @@ func TestTableMetadataToBQ(t *testing.T) {
"key1": "val1",
"key2": "val2",
},
BiglakeConfiguration: &bq.BigLakeConfiguration{
ConnectionId: "bigconn",
StorageUri: "biguri",
FileFormat: "PARQUET",
TableFormat: "ICEBERG",
},
},
},
{
Expand Down
Loading