diff --git a/.github/workflows/codeql-verification.yml b/.github/workflows/codeql-verification.yml index acc7e4fa9dd51..ea5a99dab2552 100644 --- a/.github/workflows/codeql-verification.yml +++ b/.github/workflows/codeql-verification.yml @@ -66,7 +66,7 @@ jobs: run: | set -euo pipefail ./mvnw -B \ - -pl hazelcast,hazelcast-build-utils,hazelcast-sql,extensions/cdc-debezium \ + -pl hazelcast,hazelcast-build-utils,hazelcast-sql,extensions/cdc-debezium,extensions/mongodb \ -am \ -DskipTests \ -Dcheckstyle.skip=true \ diff --git a/.github/workflows/security-release-gate.yml b/.github/workflows/security-release-gate.yml index fe64ea0341ff2..4db513f0504c0 100644 --- a/.github/workflows/security-release-gate.yml +++ b/.github/workflows/security-release-gate.yml @@ -65,7 +65,7 @@ jobs: core_tests='com.hazelcast.aws.AwsMetadataApiTest,com.hazelcast.azure.TagTest,com.hazelcast.config.XmlSchemaSourceSecurityTest,com.hazelcast.console.ConsoleAppTest,com.hazelcast.core.server.HazelcastMemberStarterTest,com.hazelcast.dataconnection.impl.JdbcDataConnectionTest,com.hazelcast.dataconnection.impl.JdbcUrlPolicyTest,com.hazelcast.dataconnection.impl.hazelcastdataconnection.HazelcastDataConnectionConfigLoaderTest,com.hazelcast.dataconnection.impl.jdbcproperties.HikariTranslatorTest,com.hazelcast.flakeidgen.impl.FlakeIdGeneratorProxyTest,com.hazelcast.gcp.LabelTest,com.hazelcast.gcp.UtilsTest,com.hazelcast.internal.config.ConfigLoaderSecurityTest,com.hazelcast.internal.diagnostics.DiagnosticsLogFileTest,com.hazelcast.internal.serialization.impl.ByteArrayObjectDataIntegrationTest,com.hazelcast.internal.serialization.impl.ObjectDataInputStreamIntegrationTest,com.hazelcast.internal.serialization.impl.UnsafeObjectDataInputIntegrationTest,com.hazelcast.internal.util.ClockTest,com.hazelcast.internal.util.HashUtilTest,com.hazelcast.internal.util.SecureFileAccessTest,com.hazelcast.internal.util.collection.LongHashSetTest,com.hazelcast.internal.util.concurrent.BackoffIdleStrategyTest,com.hazelcast.internal.util.phonehome.PhoneHomeDifferentConfigTest,com.hazelcast.internal.util.phonehome.PhoneHomeIntegrationTest,com.hazelcast.internal.util.XmlUtilTest,com.hazelcast.jet.impl.submitjob.memberside.JobUploadStatusTest,com.hazelcast.jet.impl.submitjob.memberside.validator.JarOnClientValidatorTest,com.hazelcast.jet.retry.impl.IntervalFunctionTest,com.hazelcast.kubernetes.KubernetesApiOriginPolicyTest,com.hazelcast.kubernetes.KubernetesConfigTest,com.hazelcast.spi.discovery.multicast.MulticastDiscoveryStrategyTest,com.hazelcast.spi.utils.RestClientTest,com.hazelcast.spi.utils.RetryUtilsTest' all_tests="$core_tests,com.hazelcast.jet.sql.impl.parse.QueryParserTest,com.hazelcast.buildutils.ExportPackageViewerTest,com.hazelcast.jet.cdc.OperationTest" timeout --signal=TERM --kill-after=30s 30m ./mvnw -B \ - -pl hazelcast,hazelcast-sql,hazelcast-build-utils,extensions/cdc-debezium \ + -pl hazelcast,hazelcast-sql,hazelcast-build-utils,extensions/cdc-debezium,extensions/mongodb \ -am \ -Dcheckstyle.skip=true \ -Dsurefire.failIfNoSpecifiedTests=false \ diff --git a/extensions/mongodb/src/main/java/com/hazelcast/jet/sql/impl/connector/mongodb/RexToMongoVisitor.java b/extensions/mongodb/src/main/java/com/hazelcast/jet/sql/impl/connector/mongodb/RexToMongoVisitor.java index 9f3d9f6855166..1f96e3dbbee08 100644 --- a/extensions/mongodb/src/main/java/com/hazelcast/jet/sql/impl/connector/mongodb/RexToMongoVisitor.java +++ b/extensions/mongodb/src/main/java/com/hazelcast/jet/sql/impl/connector/mongodb/RexToMongoVisitor.java @@ -22,9 +22,12 @@ import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexNodeAndFieldIndex; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexPatternFieldRef; import org.apache.calcite.rex.RexRangeRef; @@ -124,4 +127,19 @@ public Expression visitTableInputRef(RexTableInputRef fieldRef) { public Expression visitPatternFieldRef(RexPatternFieldRef fieldRef) { throw new UnsupportedOperationException(); } + + @Override + public Expression visitLambda(RexLambda lambda) { + throw new UnsupportedOperationException(); + } + + @Override + public Expression visitLambdaRef(RexLambdaRef lambdaRef) { + throw new UnsupportedOperationException(); + } + + @Override + public Expression visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) { + throw new UnsupportedOperationException(); + } } diff --git a/hazelcast-parent/pom.xml b/hazelcast-parent/pom.xml index cd060401e5f23..35e773e190f46 100644 --- a/hazelcast-parent/pom.xml +++ b/hazelcast-parent/pom.xml @@ -91,7 +91,7 @@ 2.0.3 2.42.39 - 1.35.0 + 1.42.0 + + org.apache.httpcomponents.core5 + httpcore5 + 5.3.5 + + + org.jooq + joou-java-6 + 0.9.5 + com.fasterxml.woodstox woodstox-core diff --git a/hazelcast-sql/src/main/codegen/config.fmpp b/hazelcast-sql/src/main/codegen/config.fmpp index 53f5691c54041..43ab9654d9d4d 100644 --- a/hazelcast-sql/src/main/codegen/config.fmpp +++ b/hazelcast-sql/src/main/codegen/config.fmpp @@ -540,11 +540,16 @@ data: { # Example: "< IDENTIFIER: (|)+ >". customIdentifierToken: "" + # Method for parsing "SET [OR RESET]" calls. + setOptionParserMethod: "SqlSetOption" + includePosixOperators: false includeCompoundIdentifier: true includeBraces: true includeAdditionalDeclarations: false includeParsingStringLiteralAsArrayLiteral: false + includeIntervalWithoutQualifier: false + includeStarExclude: false } } diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/JoinNestedLoopPhysicalRel.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/JoinNestedLoopPhysicalRel.java index 0a58f17e03bbd..a387f0a203f2c 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/JoinNestedLoopPhysicalRel.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/JoinNestedLoopPhysicalRel.java @@ -69,6 +69,7 @@ public JoinInfo analyzeCondition() { } List leftKeys = joinInfo.leftKeys.toIntegerList(); List rightKeys = joinInfo.rightKeys.toIntegerList(); + List nullExclusionFlags = new ArrayList<>(joinInfo.nullExclusionFlags); HazelcastTable table = OptUtils.extractHazelcastTable(getRight()); RexBuilder rexBuilder = getCluster().getRexBuilder(); @@ -92,12 +93,14 @@ public RexNode visitInputRef(RexInputRef inputRef) { rightExpr)); leftKeys.remove(i); rightKeys.remove(i); + nullExclusionFlags.remove(i); i--; } } modifiedJoinInfo = new ModifiedJoinInfo(ImmutableIntList.copyOf(leftKeys), ImmutableIntList.copyOf(rightKeys), + ImmutableList.copyOf(nullExclusionFlags), ImmutableList.builder() .addAll(joinInfo.nonEquiConditions) .addAll(additionalNonEquiConditions).build()); @@ -123,8 +126,9 @@ public Join copy( protected static class ModifiedJoinInfo extends JoinInfo { protected ModifiedJoinInfo(ImmutableIntList leftKeys, ImmutableIntList rightKeys, + ImmutableList nullExclusionFlags, ImmutableList nonEquiConditions) { - super(leftKeys, rightKeys, nonEquiConditions); + super(leftKeys, rightKeys, nullExclusionFlags, nonEquiConditions); } } } diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/visitor/RexToExpressionVisitor.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/visitor/RexToExpressionVisitor.java index f7a8584c79c81..e85c0facd2eab 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/visitor/RexToExpressionVisitor.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/opt/physical/visitor/RexToExpressionVisitor.java @@ -28,9 +28,12 @@ import org.apache.calcite.rex.RexDynamicParam; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexNodeAndFieldIndex; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexPatternFieldRef; import org.apache.calcite.rex.RexRangeRef; @@ -139,4 +142,19 @@ public Expression visitTableInputRef(RexTableInputRef fieldRef) { public Expression visitPatternFieldRef(RexPatternFieldRef fieldRef) { throw new UnsupportedOperationException(); } + + @Override + public Expression visitLambda(RexLambda lambda) { + throw new UnsupportedOperationException(); + } + + @Override + public Expression visitLambdaRef(RexLambdaRef lambdaRef) { + throw new UnsupportedOperationException(); + } + + @Override + public Expression visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) { + throw new UnsupportedOperationException(); + } } diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/schema/HazelcastDynamicTableFunction.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/schema/HazelcastDynamicTableFunction.java index 454795aee2f19..87fcd149de5f8 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/schema/HazelcastDynamicTableFunction.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/schema/HazelcastDynamicTableFunction.java @@ -22,6 +22,7 @@ import com.hazelcast.sql.impl.schema.Table; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeComparability; +import org.apache.calcite.rel.type.RelDataTypeDigest; import org.apache.calcite.rel.type.RelDataTypeFamily; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelDataTypePrecedenceList; @@ -362,5 +363,26 @@ public RelDataTypeComparability getComparability() { public boolean isDynamicStruct() { return delegate.isDynamicStruct(); } + + @Override + public RelDataTypeDigest getDigest() { + return delegate.getDigest(); + } + + @Override + public boolean deepEquals(Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof HazelcastFunctionRelDataType) { + obj = ((HazelcastFunctionRelDataType) obj).delegate; + } + return delegate.deepEquals(obj); + } + + @Override + public int deepHashCode() { + return delegate.deepHashCode(); + } } } diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/HazelcastSqlValidator.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/HazelcastSqlValidator.java index 1d77181698b92..61230406655e2 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/HazelcastSqlValidator.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/HazelcastSqlValidator.java @@ -26,7 +26,6 @@ import com.hazelcast.jet.sql.impl.schema.HazelcastTable; import com.hazelcast.jet.sql.impl.validate.HazelcastSqlOperatorTable.RewriteVisitor; import com.hazelcast.jet.sql.impl.validate.literal.LiteralUtils; -import com.hazelcast.jet.sql.impl.validate.operators.misc.HazelcastCastFunction; import com.hazelcast.jet.sql.impl.validate.param.AbstractParameterConverter; import com.hazelcast.jet.sql.impl.validate.types.HazelcastObjectType; import com.hazelcast.jet.sql.impl.validate.types.HazelcastTypeCoercion; @@ -52,7 +51,6 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlDelete; import org.apache.calcite.sql.SqlDynamicParam; -import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlIntervalLiteral; @@ -211,29 +209,6 @@ public void validateInsert(final SqlInsert insert) { validateUpsertRowType((SqlIdentifier) insert.getTargetTable()); } - @Override - public void validateColumnListParams( - final SqlFunction function, - final List argTypes, - final List operands - ) { - if (!(function instanceof HazelcastCastFunction)) { - super.validateColumnListParams(function, argTypes, operands); - } - - if (argTypes.get(0).getSqlTypeName() != SqlTypeName.COLUMN_LIST) { - throw QueryException.error("Cannot convert " + argTypes.get(0).getSqlTypeName() - + " to " + argTypes.get(1).getSqlTypeName()); - } - - final SqlCall call = (SqlCall) operands.get(0); - - assert call.getOperator().getKind() == SqlKind.ROW - : "CAST column list argument is not a RowExpression call"; - - throw QueryException.error("Cannot convert ROW to JSON"); - } - private void validateSelect(SqlSelect select, SqlValidatorScope scope) { // Derive the types for offset-fetch expressions, Calcite doesn't do // that automatically. diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/operators/misc/HazelcastCastFunction.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/operators/misc/HazelcastCastFunction.java index 5dacf37d1fc18..50e6e4fd3c559 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/operators/misc/HazelcastCastFunction.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/operators/misc/HazelcastCastFunction.java @@ -21,6 +21,7 @@ import com.hazelcast.jet.sql.impl.validate.operators.common.HazelcastFunction; import com.hazelcast.jet.sql.impl.validate.param.NoOpParameterConverter; import com.hazelcast.sql.SqlColumnType; +import com.hazelcast.sql.impl.QueryException; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCallBinding; @@ -39,6 +40,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import static com.hazelcast.jet.sql.impl.validate.types.HazelcastTypeUtils.canCast; +import static com.hazelcast.jet.sql.impl.validate.types.HazelcastTypeUtils.isJsonType; import static com.hazelcast.jet.sql.impl.validate.types.HazelcastTypeUtils.toHazelcastType; public final class HazelcastCastFunction extends HazelcastFunction { @@ -73,6 +75,13 @@ public boolean checkOperandTypes(HazelcastCallBinding binding, boolean throwOnFa binding.getValidator().setParameterConverter(sourceParameterIndex, NoOpParameterConverter.INSTANCE); } + if (sourceType.getSqlTypeName() == SqlTypeName.ROW && isJsonType(targetType)) { + if (throwOnFailure) { + throw QueryException.error("Cannot convert ROW to JSON"); + } + return false; + } + if (canCast(sourceType, targetType)) { return true; } diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastIntegerType.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastIntegerType.java index 50139dde68323..17ce52d7edb08 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastIntegerType.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastIntegerType.java @@ -125,9 +125,8 @@ public final class HazelcastIntegerType extends BasicSqlType { private final int bitWidth; private HazelcastIntegerType(SqlTypeName typeName, boolean nullable, int bitWidth) { - super(HazelcastTypeSystem.INSTANCE, typeName); + super(HazelcastTypeSystem.INSTANCE, typeName, nullable); - this.isNullable = nullable; this.bitWidth = bitWidth; // recompute the digest to reflect the nullability of the type diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastType.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastType.java index fc91417faebc6..a464b02280590 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastType.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastType.java @@ -28,9 +28,7 @@ public class HazelcastType extends BasicSqlType { } HazelcastType(SqlTypeName typeName, boolean nullable, int precision) { - super(HazelcastTypeSystem.INSTANCE, typeName, precision); - - this.isNullable = nullable; + super(HazelcastTypeSystem.INSTANCE, typeName, nullable, precision); computeDigest(); } diff --git a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastTypeSystem.java b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastTypeSystem.java index 298edee89d97a..af1d21d71bcf6 100644 --- a/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastTypeSystem.java +++ b/hazelcast-sql/src/main/java/com/hazelcast/jet/sql/impl/validate/types/HazelcastTypeSystem.java @@ -57,13 +57,13 @@ private HazelcastTypeSystem() { } @Override - public int getMaxNumericPrecision() { - return MAX_DECIMAL_PRECISION; + public int getMaxPrecision(SqlTypeName typeName) { + return typeName == DECIMAL ? MAX_DECIMAL_PRECISION : super.getMaxPrecision(typeName); } @Override - public int getMaxNumericScale() { - return MAX_DECIMAL_SCALE; + public int getMaxScale(SqlTypeName typeName) { + return typeName == DECIMAL ? MAX_DECIMAL_SCALE : super.getMaxScale(typeName); } @Override