Skip to content

feat(bigquery-jdbc): add graph integration in getColumns method #14893

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Neenu1995 wants to merge 11 commits into from
Open

feat(bigquery-jdbc): add graph integration in getColumns method #14893

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
0f91838
Merge branch 'main' into jdbc/graph-get-columns
Neenu1995 Nov 2, 2026
7cb03ce
feat(bigquery-jdbc): add graph integration in getColumns method
Neenu1995 Nov 2, 2026
c9fd433
resolve conflicts
Neenu1995 Nov 2, 2026
8a5b9a8
address comment
Neenu1995 Nov 2, 2026
6c5a845
Merge branch 'jdbc/graph-get-tables' into jdbc/graph-get-columns
Neenu1995 Nov 3, 2026
9fd7af1
lint
Neenu1995 Nov 3, 2026
7112cee
fix tests
Neenu1995 Nov 3, 2026
ea06745
change column aliases
Neenu1995 Nov 3, 2026
79dae13
cleanup
Neenu1995 Nov 3, 2026
f4e4c13
Merge branch 'jdbc/graph-get-tables' into jdbc/graph-get-columns
Neenu1995 Nov 3, 2026
97bde46
identifier validation
Neenu1995 Nov 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
267 changes: 221 additions & 46 deletions ...-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,10 @@ class BigQueryDatabaseMetaData implements DatabaseMetaData {
private static String exportedKeysSqlContent;
private static final String GET_GRAPH_TABLES_SQL = "DatabaseMetaData_GetGraphTables.sql";
private static String graphTablesSqlContent;
private static final String GET_GRAPH_COLUMNS_SQL = "DatabaseMetaData_GetGraphColumns.sql";
private static String graphColumnsSqlContent;
static final String MEASURE_EXPRESSION_KIND = "MEASURE";
static final String MEASURE_REMARKS_PREFIX = "MEASURE: ";
// Declared package-private for testing.
static final String GOOGLE_SQL_QUOTED_IDENTIFIER = "`";
// Does not include SQL:2003 Keywords as per JDBC spec.
Expand Down Expand Up @@ -1233,25 +1237,16 @@ ColumnTypeInfo determineTypeInfoFromDataType(
String procedureName,
String columnName,
int ordinalPosition) {

ColumnTypeInfo defaultStringTypeInfo =
new ColumnTypeInfo(Types.NVARCHAR, "STRING", null, null, null);
try {
String typeKind = argumentDataType.getTypeKind();
if (typeKind != null && !typeKind.isEmpty()) {
if ("ARRAY".equalsIgnoreCase(typeKind)) {
return new ColumnTypeInfo(Types.ARRAY, "ARRAY", null, null, null);
}
StandardSQLTypeName determinedType = StandardSQLTypeName.valueOf(typeKind.toUpperCase());
return getColumnTypeInfoForSqlType(determinedType);
}
return getColumnTypeInfoForTypeKind(
argumentDataType != null ? argumentDataType.getTypeKind() : null);
} catch (Exception e) {
LOG.warning(
"Proc: %s, Arg: %s (Pos %d) - Caught an unexpected Exception during type"
+ " determination. Defaulting type to STRING. Error: %s",
procedureName, columnName, ordinalPosition, e.getMessage());
return getColumnTypeInfoForSqlType(null);
}
return defaultStringTypeInfo;
}

Comparator<FieldValueList> defineGetProcedureColumnsComparator(FieldList resultSchemaFields) {
Expand Down Expand Up @@ -1502,10 +1497,14 @@ static void validateRegionScopeIdentifier(String regionScope) throws SQLExceptio
}

boolean shouldSkipGraphTables(String tableNamePattern, Set<String> requestedTypes) {
if (this.connection == null || !this.connection.isEnableGraphMetadata()) {
if (requestedTypes != null && !requestedTypes.contains(GRAPH_TABLE_TYPE)) {
return true;
}
if (requestedTypes != null && !requestedTypes.contains(GRAPH_TABLE_TYPE)) {
return shouldSkipGraphColumns(tableNamePattern);
}

boolean shouldSkipGraphColumns(String tableNamePattern) {
if (this.connection == null || !this.connection.isEnableGraphMetadata()) {
return true;
}
if (tableNamePattern != null
Expand Down Expand Up @@ -1716,19 +1715,32 @@ private ResultSet getColumnsImpl(
() -> {
final List<FieldValueList> collectedResults =
Collections.synchronizedList(new ArrayList<>());
processTargetTablesConcurrently(
targetDatasets,
boolean isExactGraphTableName =
tableNamePattern != null
&& !tableNamePattern.contains("%")
&& tableNamePattern.contains(".");
if (!isExactGraphTableName) {
processTargetTablesConcurrently(
targetDatasets,
tableNamePattern,
true,
false,
true,
collectedResults,
resultSchemaFields,
(bqTable, results, fields) -> {
processTableColumns(bqTable, columnNameRegex, results, fields);
},
TableOption.fields(
TableField.TABLE_REFERENCE, TableField.TYPE, TableField.SCHEMA));
}
appendGraphColumnRows(
effectiveCatalog,
effectiveSchemaPattern,
tableNamePattern,
true,
false,
true,
columnNamePattern,
collectedResults,
resultSchemaFields,
(bqTable, results, fields) -> {
processTableColumns(bqTable, columnNameRegex, results, fields);
},
TableOption.fields(
TableField.TABLE_REFERENCE, TableField.TYPE, TableField.SCHEMA));
resultSchemaFields);

Comparator<FieldValueList> comparator =
defineGetColumnsComparator(resultSchemaFields);
Expand Down Expand Up @@ -1807,7 +1819,141 @@ private void processTableColumns(
}
}

private Schema defineGetColumnsSchema() {
void appendGraphColumnRows(
String effectiveCatalog,
String effectiveSchemaPattern,
String tableNamePattern,
String columnNamePattern,
List<FieldValueList> collectedResults,
FieldList resultSchemaFields)
throws SQLException {
if (shouldSkipGraphColumns(tableNamePattern)) {
return;
}

LOG.info("Appending Graph column rows");

List<String> targetCatalogs =
effectiveCatalog != null
? Collections.singletonList(effectiveCatalog)
: getAccessibleCatalogNames();
boolean isExactDataset =
effectiveSchemaPattern != null && !effectiveSchemaPattern.contains("%");
String effectiveSchemaLike = (effectiveSchemaPattern == null) ? "%" : effectiveSchemaPattern;
String effectiveTableLike = (tableNamePattern == null) ? "%" : tableNamePattern;
String effectiveColumnLike = (columnNamePattern == null) ? "%" : columnNamePattern;
for (String project : targetCatalogs) {
validateCatalogIdentifier(project);
String scope = isExactDataset ? effectiveSchemaPattern : resolveGraphRegion();

if (isExactDataset) {
validateSchemaIdentifier(scope);
} else {
validateRegionScopeIdentifier(scope);
}

String formattedSql = replaceSqlParameters(getGraphColumnsSqlContent(), project, scope);
try (PreparedStatement statement = this.connection.prepareStatement(formattedSql)) {
statement.setString(1, effectiveSchemaLike);
statement.setString(2, effectiveTableLike);
statement.setString(3, effectiveColumnLike);
try (ResultSet rs = statement.executeQuery()) {
while (rs.next()) {
String tableCat = rs.getString("TABLE_CAT");
String tableSchem = rs.getString("TABLE_SCHEM");
String tableName = rs.getString("TABLE_NAME");
String columnName = rs.getString("COLUMN_NAME");
String typeKind =
rs.getString("TYPE_KIND"); // DATA_TYPE, TYPE_NAME, COLUMN_SIZE, DECIMAL_DIGITS,
// NUM_PREC_RADIX
String expression =
rs.getString("EXPRESSION"); // REMARKS (for measures) & IS_GENERATEDCOLUMN
String expressionKind =
rs.getString(
"EXPRESSION_KIND"); // REMARKS ("MEASURE: " prefix) & IS_GENERATEDCOLUMN
String description = rs.getString("DESCRIPTION"); // REMARKS (property description)
int ordinalPosition = rs.getInt("ORDINAL_POSITION");
List<FieldValue> values =
createGraphColumnRow(
tableCat,
tableSchem,
tableName,
columnName,
typeKind,
expression,
expressionKind,
description,
ordinalPosition);
collectedResults.add(FieldValueList.of(values, resultSchemaFields));
}
}
} catch (SQLException e) {
if (e.getMessage() != null && e.getMessage().contains("Not found")) {
LOG.info(
"Dataset or project not found while querying PROPERTY_GRAPHS for columns in project"
+ " '%s'. Skipping.",
project);
} else {
throw e;
}
}
}
}

List<FieldValue> createGraphColumnRow(
String catalog,
String schemaName,
String tableName,
String columnName,
String typeKind,
@Nullable String expression,
@Nullable String expressionKind,
@Nullable String description,
int ordinalPosition) {
ColumnTypeInfo typeInfo = getColumnTypeInfoForTypeKind(typeKind);

boolean isMeasure = MEASURE_EXPRESSION_KIND.equals(expressionKind);
boolean isComputedExpression =
expression != null && !expression.isEmpty() && !expression.equalsIgnoreCase(columnName);
String isGeneratedColumn = (isMeasure || isComputedExpression) ? "YES" : "NO";

String remarks;
if (isMeasure) {
String measurePrefix = MEASURE_REMARKS_PREFIX + (expression != null ? expression : "");
remarks =
(description == null || description.isEmpty())
? measurePrefix
: measurePrefix + " - " + description;
} else {
remarks = description;
}

return buildColumnRowValues(
catalog,
schemaName,
tableName,
columnName,
typeInfo,
DatabaseMetaData.columnNullableUnknown,
remarks,
ordinalPosition,
"",
isGeneratedColumn);
}

ColumnTypeInfo getColumnTypeInfoForTypeKind(@Nullable String typeKind) {
if (typeKind == null || typeKind.isEmpty()) {
return getColumnTypeInfoForSqlType(null);
}
try {
return getColumnTypeInfoForSqlType(StandardSQLTypeName.valueOf(typeKind));
} catch (IllegalArgumentException e) {
LOG.warning("Unknown BigQuery typeKind '%s' in graph metadata. Mapping to STRING.", typeKind);
return getColumnTypeInfoForSqlType(null);
}
}
Comment thread
Neenu1995 marked this conversation as resolved.
Resolved

Schema defineGetColumnsSchema() {
List<Field> fields = new ArrayList<>(24);
fields.add(
Field.newBuilder("TABLE_CAT", StandardSQLTypeName.STRING)
Expand Down Expand Up @@ -1910,14 +2056,51 @@ private Schema defineGetColumnsSchema() {

List<FieldValue> createColumnRow(
String catalog, String schemaName, String tableName, Field field, int ordinalPosition) {
List<FieldValue> values = new ArrayList<>(24);
Field.Mode mode = (field.getMode() == null) ? Field.Mode.NULLABLE : field.getMode();
ColumnTypeInfo typeInfo = mapBigQueryTypeToJdbc(field);

int nullable =
(mode == Field.Mode.REQUIRED)
? DatabaseMetaData.columnNoNulls
: DatabaseMetaData.columnNullable;
String isNullable = "";
switch (mode) {
case REQUIRED:
isNullable = "NO";
break;
case NULLABLE:
case REPEATED:
isNullable = "YES";
break;
}
return buildColumnRowValues(
catalog,
schemaName,
tableName,
field.getName(),
typeInfo,
nullable,
field.getDescription(),
ordinalPosition,
isNullable,
"NO");
}

private List<FieldValue> buildColumnRowValues(
String catalog,
String schemaName,
String tableName,
String columnName,
ColumnTypeInfo typeInfo,
int nullable,
@Nullable String remarks,
int ordinalPosition,
String isNullable,
String isGeneratedColumn) {
List<FieldValue> values = new ArrayList<>(24);
values.add(createStringFieldValue(catalog)); // 1. TABLE_CAT
values.add(createStringFieldValue(schemaName)); // 2. TABLE_SCHEM
values.add(createStringFieldValue(tableName)); // 3. TABLE_NAME
values.add(createStringFieldValue(field.getName())); // 4. COLUMN_NAME
values.add(createStringFieldValue(columnName)); // 4. COLUMN_NAME
values.add(createLongFieldValue((long) typeInfo.jdbcType)); // 5. DATA_TYPE
values.add(createStringFieldValue(typeInfo.typeName)); // 6. TYPE_NAME
values.add(
Expand All @@ -1936,35 +2119,20 @@ List<FieldValue> createColumnRow(
typeInfo.numPrecRadix == null
? null
: typeInfo.numPrecRadix.longValue())); // 10. NUM_PREC_RADIX
int nullable =
(mode == Field.Mode.REQUIRED)
? DatabaseMetaData.columnNoNulls
: DatabaseMetaData.columnNullable;
values.add(createLongFieldValue((long) nullable)); // 11. NULLABLE
values.add(createStringFieldValue(field.getDescription())); // 12. REMARKS
values.add(createStringFieldValue(remarks)); // 12. REMARKS
values.add(createNullFieldValue()); // 13. COLUMN_DEF
values.add(createNullFieldValue()); // 14. SQL_DATA_TYPE
values.add(createNullFieldValue()); // 15. SQL_DATETIME_SUB
values.add(createNullFieldValue()); // 16. CHAR_OCTET_LENGTH
values.add(createLongFieldValue((long) ordinalPosition)); // 17. ORDINAL_POSITION
String isNullable = "";
switch (mode) {
case REQUIRED:
isNullable = "NO";
break;
case NULLABLE:
case REPEATED:
isNullable = "YES";
break;
}
values.add(createStringFieldValue(isNullable)); // 18. IS_NULLABLE
values.add(createNullFieldValue()); // 19. SCOPE_CATALOG
values.add(createNullFieldValue()); // 20. SCOPE_SCHEMA
values.add(createNullFieldValue()); // 21. SCOPE_TABLE
values.add(createNullFieldValue()); // 22. SOURCE_DATA_TYPE
values.add(createStringFieldValue("NO")); // 23. IS_AUTOINCREMENT
values.add(createStringFieldValue("NO")); // 24. IS_GENERATEDCOLUMN

values.add(createStringFieldValue(isGeneratedColumn)); // 24. IS_GENERATEDCOLUMN
return values;
}

Expand Down Expand Up @@ -5023,6 +5191,13 @@ static synchronized String getGraphTablesSqlContent() {
return graphTablesSqlContent;
}

private static synchronized String getGraphColumnsSqlContent() {
if (graphColumnsSqlContent == null) {
graphColumnsSqlContent = readSqlFromFile(GET_GRAPH_COLUMNS_SQL);
}
return graphColumnsSqlContent;
}

static String readSqlFromFile(String filename) {
try (InputStream in = BigQueryDatabaseMetaData.class.getResourceAsStream(filename)) {
if (in == null) {
Expand Down
Loading
Loading