From 6cca1f465399e203e6a7e3dd11836cbc87f975e9 Mon Sep 17 00:00:00 2001 From: korlov42 Date: Wed, 15 Jan 2025 16:25:40 +0200 Subject: [PATCH] IGNITE-24167 Sql. Introduce heuristics to optimize join order (#5026) --- .../internal/index/ItBuildIndexTest.java | 2 +- .../benchmark/AbstractTpcBenchmark.java | 9 + .../internal/sql/engine/ItAggregatesTest.java | 4 +- .../internal/sql/engine/ItJoinOrderTest.java | 315 ++++++++++++ .../sql/engine/ItOrToUnionRuleTest.java | 4 +- .../sql/engine/ItSecondaryIndexTest.java | 4 + .../tests/BaseIndexDataTypeTest.java | 4 +- .../sql/engine/statistic/ItStatisticTest.java | 4 +- .../group1/join/test_not_distinct_from.test | 8 +- .../sql/engine/metadata/IgniteMdRowCount.java | 4 +- .../sql/engine/prepare/IgnitePlanner.java | 55 ++- .../sql/engine/prepare/PlannerHelper.java | 95 +++- .../sql/engine/prepare/PlannerPhase.java | 58 ++- .../sql/engine/prepare/PlanningContext.java | 28 +- .../engine/rule/HashJoinConverterRule.java | 12 +- .../engine/rule/MergeJoinConverterRule.java | 6 +- .../rule/NestedLoopJoinConverterRule.java | 12 +- .../IgniteMultiJoinOptimizeBushyRule.java | 456 ++++++++++++++++++ .../statistic/SqlStatisticManagerImpl.java | 32 +- .../sql/engine/framework/TestBuilders.java | 20 +- .../engine/planner/AbstractPlannerTest.java | 2 +- .../planner/AbstractTpcQueryPlannerTest.java | 156 ++++++ .../planner/JoinCommutePlannerTest.java | 337 ------------- .../engine/planner/TpcdsQueryPlannerTest.java | 69 +++ .../engine/planner/TpchQueryPlannerTest.java | 93 +--- .../SqlStatisticManagerImplTest.java | 16 - .../resources/mapping/table_identity.test | 82 ++-- .../mapping/table_identity_single.test | 60 +-- .../test/resources/mapping/table_single.test | 32 +- .../mapping/test_partition_pruning.test | 25 +- .../src/test/resources/tpcds/plan/q64.plan | 127 +++++ .../src/test/resources/tpch/plan/q5.plan | 20 + .../src/test/resources/tpch/plan/q7.plan | 20 + .../src/test/resources/tpch/plan/q8.plan | 27 ++ .../src/test/resources/tpch/plan/q9.plan | 20 + .../internal/sql/BaseSqlIntegrationTest.java | 16 + .../sql/engine/util/TpcScaleFactor.java | 30 ++ .../internal/sql/engine/util/TpcTable.java | 3 + .../sql/engine/util/tpcds/TpcdsTables.java | 34 +- .../sql/engine/util/tpch/TpchTables.java | 18 +- .../testFixtures/resources/tpcds/query64.sql | 4 +- .../src/testFixtures/resources/tpch/q5.sql | 1 + .../src/testFixtures/resources/tpch/q7.sql | 1 + .../src/testFixtures/resources/tpch/q8.sql | 1 + .../src/testFixtures/resources/tpch/q9.sql | 1 + 45 files changed, 1719 insertions(+), 608 deletions(-) create mode 100644 modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinOrderTest.java create mode 100644 modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/IgniteMultiJoinOptimizeBushyRule.java create mode 100644 modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractTpcQueryPlannerTest.java delete mode 100644 modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/JoinCommutePlannerTest.java create mode 100644 modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpcdsQueryPlannerTest.java create mode 100644 modules/sql-engine/src/test/resources/tpcds/plan/q64.plan create mode 100644 modules/sql-engine/src/test/resources/tpch/plan/q5.plan create mode 100644 modules/sql-engine/src/test/resources/tpch/plan/q7.plan create mode 100644 modules/sql-engine/src/test/resources/tpch/plan/q8.plan create mode 100644 modules/sql-engine/src/test/resources/tpch/plan/q9.plan create mode 100644 modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcScaleFactor.java diff --git a/modules/index/src/integrationTest/java/org/apache/ignite/internal/index/ItBuildIndexTest.java b/modules/index/src/integrationTest/java/org/apache/ignite/internal/index/ItBuildIndexTest.java index ee9525822a5..33ed44acbb5 100644 --- a/modules/index/src/integrationTest/java/org/apache/ignite/internal/index/ItBuildIndexTest.java +++ b/modules/index/src/integrationTest/java/org/apache/ignite/internal/index/ItBuildIndexTest.java @@ -102,7 +102,7 @@ void testBuildIndexOnStableTopology(int replicas) throws Exception { checkIndexBuild(partitions, replicas, INDEX_NAME); - assertQuery(format("SELECT * FROM {} WHERE i1 > 0", TABLE_NAME)) + assertQuery(format("SELECT /*+ FORCE_INDEX({}) */ * FROM {} WHERE i1 > 0", INDEX_NAME, TABLE_NAME)) .matches(containsIndexScan("PUBLIC", TABLE_NAME, INDEX_NAME)) .returns(1, 1) .returns(2, 2) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/AbstractTpcBenchmark.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/AbstractTpcBenchmark.java index 8ec3b5ab51a..9798e2121ca 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/AbstractTpcBenchmark.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/AbstractTpcBenchmark.java @@ -22,8 +22,11 @@ import java.nio.file.Path; import java.time.Duration; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import org.apache.ignite.internal.sql.engine.SqlQueryProcessor; +import org.apache.ignite.internal.sql.engine.statistic.SqlStatisticManagerImpl; import org.apache.ignite.internal.sql.engine.util.TpcTable; import org.apache.ignite.sql.IgniteSql; import org.openjdk.jmh.annotations.Scope; @@ -70,6 +73,12 @@ public void initSchema() throws Throwable { Files.createFile(workDir().resolve(DATASET_READY_MARK_FILE_NAME)); } + + SqlStatisticManagerImpl statisticManager = (SqlStatisticManagerImpl) ((SqlQueryProcessor) igniteImpl.queryEngine()) + .sqlStatisticManager(); + + statisticManager.forceUpdateAll(); + statisticManager.lastUpdateStatisticFuture().get(10, TimeUnit.SECONDS); } catch (Throwable e) { nodeTearDown(); diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItAggregatesTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItAggregatesTest.java index 7b6b3773932..ef1810af9cd 100644 --- a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItAggregatesTest.java +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItAggregatesTest.java @@ -115,6 +115,8 @@ static void initTestData() { + "int_col INTEGER NOT NULL, " + "dec4_2_col DECIMAL(4,2) NOT NULL" + ")"); + + gatherStatistics(); } @ParameterizedTest @@ -332,7 +334,7 @@ public void testColocatedAggregate() { assertQuery(sql) .disableRules("HashJoinConverter", "MergeJoinConverter") - .matches(QueryChecker.matches(".*Join.*Exchange.*Scan.*Exchange.*Colocated.*Aggregate.*")) + .matches(QueryChecker.matches(".*Exchange.*Join.*Colocated.*Aggregate.*")) .returns("val0", 50L) .returns("val1", 50L) .check(); diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinOrderTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinOrderTest.java new file mode 100644 index 00000000000..97af092e487 --- /dev/null +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinOrderTest.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Supplier; +import org.apache.ignite.internal.sql.BaseSqlIntegrationTest; +import org.apache.ignite.internal.sql.engine.util.QueryChecker; +import org.apache.ignite.internal.util.ArrayUtils; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +/** + * Test to make sure JOIN ORDER optimization returns equivalent plan, i.e. plan returning equal result set. + */ +public class ItJoinOrderTest extends BaseSqlIntegrationTest { + private static final int PRODUCTS = 5000; + + @BeforeAll + public static void initSchema() { + int users = 10000; + int orders = 20000; + int orderDetails = 100000; + int categories = 100; + int reviews = 50000; + int discounts = 2000; + int warehouses = 50; + int shippings = 15000; + + sql("CREATE TABLE Users (\n" + + " UserID INT PRIMARY KEY,\n" + + " UserName VARCHAR(100),\n" + + " UserEmail VARCHAR(100)\n" + + ");" + ); + + sql("INSERT INTO Users SELECT x, 'User_' || x::VARCHAR, 'user' || x::VARCHAR || '@example.com' " + + "FROM system_range(1, ?)", users); + + sql("CREATE TABLE Orders (\n" + + " OrderID INT PRIMARY KEY,\n" + + " UserID INT,\n" + + " OrderDate DATE,\n" + + " TotalAmount DECIMAL(10, 2)\n" + + ");" + ); + + sql("INSERT INTO Orders SELECT x, 1 + RAND_INTEGER(?), date '2020-01-01' + RAND_INTEGER(365)::INTERVAL DAYS, " + + "ROUND(50.0 + 1950.0 * RAND(), 2) FROM system_range(1, ?)", users - 1, orders); + + sql("CREATE TABLE Products (\n" + + " ProductID INT PRIMARY KEY,\n" + + " ProductName VARCHAR(100),\n" + + " Price DECIMAL(10, 2)\n" + + ");" + ); + + sql("INSERT INTO Products SELECT x, 'Product_' || x::VARCHAR, " + + "ROUND(5.0 + 495.0 * RAND(), 2) FROM system_range(1, ?)", PRODUCTS); + + sql("CREATE TABLE OrderDetails (\n" + + " OrderDetailID INT PRIMARY KEY,\n" + + " OrderID INT,\n" + + " ProductID INT,\n" + + " Quantity INT\n" + + ");" + ); + + sql("INSERT INTO OrderDetails SELECT x, 1 + RAND_INTEGER(?), 1 + RAND_INTEGER(?), " + + "1 + RAND_INTEGER(9) FROM system_range(1, ?)", orders - 1, PRODUCTS - 1, orderDetails); + + sql("CREATE TABLE Categories (\n" + + " CategoryID INT PRIMARY KEY,\n" + + " CategoryName VARCHAR(100)\n" + + ");" + ); + + sql("INSERT INTO Categories SELECT x, 'Category_' || x::VARCHAR FROM system_range(1, ?)", categories); + + sql("CREATE TABLE ProductCategories (\n" + + " ProductCategoryID INT PRIMARY KEY,\n" + + " ProductID INT,\n" + + " CategoryID INT\n" + + ");" + ); + + sql("INSERT INTO ProductCategories SELECT x, 1 + RAND_INTEGER(?)," + + " 1 + RAND_INTEGER(?) FROM system_range(1, ?)", PRODUCTS - 1, categories - 1, PRODUCTS); + + sql("CREATE TABLE Shipping (\n" + + " ShippingID INT PRIMARY KEY,\n" + + " OrderID INT,\n" + + " ShippingDate DATE,\n" + + " ShippingAddress VARCHAR(255)\n" + + ");" + ); + + sql("INSERT INTO Shipping SELECT x, 1 + RAND_INTEGER(?), date '2020-01-01' + RAND_INTEGER(365)::INTERVAL DAYS, " + + " 'Address_' || x::VARCHAR FROM system_range(1, ?)", orders - 1, shippings); + + sql("CREATE TABLE Reviews (\n" + + " ReviewID INT PRIMARY KEY,\n" + + " ProductID INT,\n" + + " UserID INT,\n" + + " ReviewText VARCHAR,\n" + + " Rating INT\n" + + ");" + ); + + sql("INSERT INTO Reviews SELECT x, 1 + RAND_INTEGER(?), 1 + RAND_INTEGER(?)" + + ", 'This is a review for product ' || x::VARCHAR, 1 + RAND_INTEGER(4) FROM system_range(1, ?)", + PRODUCTS - 1, users - 1, reviews); + + sql("CREATE TABLE Discounts (\n" + + " DiscountID INT PRIMARY KEY,\n" + + " ProductID INT,\n" + + " DiscountPercentage DECIMAL(5, 2),\n" + + " ValidUntil DATE\n" + + ");" + ); + + sql("INSERT INTO Discounts SELECT x, 1 + RAND_INTEGER(?), ROUND(5.0 + 45.0 * RAND(), 2) " + + ", date '2020-01-01' + RAND_INTEGER(365)::INTERVAL DAYS FROM system_range(1, ?)", + PRODUCTS - 1, discounts); + + sql("CREATE TABLE Warehouses (\n" + + " WarehouseID INT PRIMARY KEY,\n" + + " WarehouseName VARCHAR(100),\n" + + " Location VARCHAR(100)\n" + + ");" + ); + + sql("INSERT INTO Warehouses SELECT x, 'Warehouse_' || x::VARCHAR, " + + "'Location_' || x::VARCHAR FROM system_range(1, ?)", warehouses); + + gatherStatistics(); + } + + @ParameterizedTest + @EnumSource(Query.class) + void test(Query query) { + String originalText = query.text(); + String textWithEnforcedJoinOrder = originalText + .replace("SELECT", "SELECT /*+ enforce_join_order */ "); + + Object[] params = query.params(); + + List> expectedResult = sql(textWithEnforcedJoinOrder, params); + + Assumptions.assumeFalse(expectedResult.isEmpty()); + + QueryChecker checker = assertQuery(originalText) + .withParams(params); + + expectedResult.forEach(row -> checker.returns(row.toArray())); + + checker.check(); + } + + enum Query { + ORDERS_WITH_TOTAL_REVENUE_AND_SHIPPING_DETAILS( + "SELECT \n" + + " O.OrderID, O.OrderDate, S.ShippingAddress, SUM(OD.Quantity * P.Price) AS TotalOrderValue\n" + + " FROM Orders O, Shipping S, OrderDetails OD, Products P\n" + + "WHERE O.OrderID = S.OrderID\n" + + " AND O.OrderID = OD.OrderID\n" + + " AND OD.ProductID = P.ProductID\n" + + "GROUP BY O.OrderID, O.OrderDate, S.ShippingAddress;" + ), + + TOP_RATED_PRODUCTS_AND_THEIR_REVIEWERS( + "SELECT \n" + + " P.ProductName, R.Rating, U.UserName, R.ReviewText\n" + + " FROM Products P, Reviews R, Users U\n" + + "WHERE P.ProductID = R.ProductID\n" + + " AND R.UserID = U.UserID\n" + + " AND R.Rating IN (4, 5);" + ), + + USER_ORDERS_WITH_PRODUCTS_IN_MULTIPLE_CATEGORIES( + "SELECT \n" + + " U.UserName, O.OrderID, COUNT(DISTINCT C.CategoryName) AS Categories\n" + + " FROM Users U, Orders O, OrderDetails OD, Products P, ProductCategories PC, Categories C\n" + + "WHERE U.UserID = O.UserID\n" + + " AND O.OrderID = OD.OrderID\n" + + " AND OD.ProductID = P.ProductID\n" + + " AND P.ProductID = PC.ProductID\n" + + " AND PC.CategoryID = C.CategoryID\n" + + "GROUP BY U.UserName, O.OrderID;" + ), + + PRODUCTS_STORED_IN_WAREHOUSES_BY_CATEGORY( + "SELECT \n" + + " W.WarehouseName, C.CategoryName, P.ProductName\n" + + " FROM Warehouses W, Products P, ProductCategories PC, Categories C\n" + + "WHERE W.WarehouseID = (P.ProductID % 5 + 1)\n" + + " AND P.ProductID = PC.ProductID\n" + + " AND PC.CategoryID = C.CategoryID;" + ), + + //CHECKSTYLE:OFF + // For some reason checkstyle fails here with 'lambda arguments' has incorrect indentation level 16, expected level should be 32 + USERS_WHO_HAVE_WRITTEN_REVIEWS_FOR_A_SPECIFIC_PRODUCT( + "SELECT \n" + + " U.UserName, P.ProductName, R.ReviewText, R.Rating\n" + + " FROM Users U, Reviews R, Products P\n" + + "WHERE U.UserID = R.UserID\n" + + " AND R.ProductID = P.ProductID\n" + + " AND P.ProductName = 'Product_' || ?::varchar;", + () -> new Object[]{ThreadLocalRandom.current().nextInt(1, PRODUCTS)} + ), + //CHECKSTYLE:ON + + LIST_OF_PRODUCTS_WITH_DISCOUNTS_APPLIED_AND_THEIR_FINAL_PRICES( + "SELECT \n" + + " P.ProductName, P.Price, D.DiscountPercentage, \n" + + " (P.Price * (1 - D.DiscountPercentage / 100)) AS FinalPrice\n" + + " FROM Products P, Discounts D\n" + + "WHERE P.ProductID = D.ProductID;" + ), + + ORDERS_SHIPPED_WITH_TOTAL_QUANTITY_AND_SHIPPING_ADDRESS( + "SELECT \n" + + " O.OrderID, O.OrderDate, S.ShippingAddress, SUM(OD.Quantity) AS TotalQuantity\n" + + " FROM Orders O, Shipping S, OrderDetails OD\n" + + "WHERE O.OrderID = S.OrderID\n" + + " AND O.OrderID = OD.OrderID\n" + + "GROUP BY O.OrderID, O.OrderDate, S.ShippingAddress;" + ), + + AVERAGE_RATING_OF_PRODUCTS_IN_EACH_CATEGORY( + "SELECT \n" + + " C.CategoryName, P.ProductName, AVG(R.Rating) AS AvgRating\n" + + " FROM Categories C, ProductCategories PC, Products P, Reviews R\n" + + "WHERE C.CategoryID = PC.CategoryID\n" + + " AND PC.ProductID = P.ProductID\n" + + " AND P.ProductID = R.ProductID\n" + + "GROUP BY C.CategoryName, P.ProductName;" + ), + + PRODUCTS_ORDERED_BY_EACH_USER( + "SELECT \n" + + " U.UserName, P.ProductName, SUM(OD.Quantity) AS TotalQuantity\n" + + " FROM Users U, Orders O, OrderDetails OD, Products P\n" + + "WHERE U.UserID = O.UserID\n" + + " AND O.OrderID = OD.OrderID\n" + + " AND OD.ProductID = P.ProductID\n" + + "GROUP BY U.UserName, P.ProductName;" + ), + + TOTAL_REVENUE_GENERATED_BY_EACH_USER( + "SELECT \n" + + " U.UserID, U.UserName, SUM(O.TotalAmount) AS TotalRevenue\n" + + " FROM Users U, Orders O\n" + + "WHERE U.UserID = O.UserID\n" + + "GROUP BY U.UserID, U.UserName;" + ), + + JOIN_ALL_TABLES( + "SELECT \n" + + " U.UserID, U.UserName, O.OrderID, O.OrderDate, P.ProductName, OD.Quantity, \n" + + " C.CategoryName, S.ShippingAddress, R.Rating, D.DiscountPercentage, W.WarehouseName\n" + + " FROM Users U, Orders O, OrderDetails OD, Products P, ProductCategories PC, Categories C, \n" + + " Shipping S, Reviews R, Discounts D, Warehouses W\n" + + "WHERE U.UserID = O.UserID\n" + + " AND O.OrderID = OD.OrderID\n" + + " AND OD.ProductID = P.ProductID\n" + + " AND P.ProductID = PC.ProductID\n" + + " AND PC.CategoryID = C.CategoryID\n" + + " AND O.OrderID = S.OrderID\n" + + " AND P.ProductID = R.ProductID" + + " AND U.UserID = R.UserID\n" + + " AND P.ProductID = D.ProductID\n" + + " AND W.WarehouseID = (P.ProductID % 5 + 1);" + ); + + private final String text; + private final Supplier paramsSupplier; + + Query(String text) { + this(text, () -> ArrayUtils.OBJECT_EMPTY_ARRAY); + } + + Query(String text, Supplier paramsSupplier) { + this.text = text; + this.paramsSupplier = paramsSupplier; + } + + String text() { + return text; + } + + Object[] params() { + return paramsSupplier.get(); + } + } +} diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItOrToUnionRuleTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItOrToUnionRuleTest.java index 4e82af3e08b..735c7b40a07 100644 --- a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItOrToUnionRuleTest.java +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItOrToUnionRuleTest.java @@ -33,7 +33,7 @@ * *

Example: SELECT * FROM products WHERE category = 'Photo' OR subcategory ='Camera Media'; * - *

A query above will be rewritten to next (or equivalient similar query) + *

A query above will be rewritten to next (or equivalent similar query) * *

SELECT * FROM products WHERE category = 'Photo' UNION ALL SELECT * FROM products WHERE subcategory ='Camera Media' AND LNNVL(category, * 'Photo'); @@ -85,6 +85,8 @@ static void initTestData() { {22, null, 0, null, 40, null}, {23, null, 0, null, 41, null}, }); + + gatherStatistics(); } /** diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItSecondaryIndexTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItSecondaryIndexTest.java index af238b68c26..3b316aa0ef1 100644 --- a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItSecondaryIndexTest.java +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItSecondaryIndexTest.java @@ -121,6 +121,8 @@ static void initTestData() { {6, 6}, {7, null} }); + + gatherStatistics(); } @Test @@ -747,6 +749,7 @@ public void testSelectWithRanges() { @Test public void testIndexedNullableFieldGreaterThanFilter() { assertQuery("SELECT * FROM T1 WHERE val > 4") + .disableRules("LogicalTableScanConverterRule") .matches(containsIndexScan("PUBLIC", "T1", "T1_IDX")) .returns(5, 5) .returns(6, 6) @@ -815,6 +818,7 @@ public void testIndexBoundsMerge() { @Test public void testIndexedNullableFieldLessThanFilter() { assertQuery("SELECT * FROM T1 WHERE val <= 5") + .disableRules("LogicalTableScanConverterRule") .matches(containsIndexScan("PUBLIC", "T1", "T1_IDX")) .returns(3, 3) .returns(4, 4) diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/datatypes/tests/BaseIndexDataTypeTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/datatypes/tests/BaseIndexDataTypeTest.java index 3c2b9d2d1eb..409b54fde7a 100644 --- a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/datatypes/tests/BaseIndexDataTypeTest.java +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/datatypes/tests/BaseIndexDataTypeTest.java @@ -211,7 +211,9 @@ public void testCompoundIndex(TestTypeArguments arguments) throws Interrupted sql("drop index if exists t_test_key_pk_idx"); sql("create index if not exists t_test_key_pk_idx on t (test_key, id)"); - runSql("insert into t values(100, $0)"); + runSql("insert into t SELECT x, $0 FROM system_range(10, 100)"); + + gatherStatistics(); String query = format("select id, test_key from t where test_key = {} and id >= 100", arguments.valueExpr(0)); checkQuery(query) diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/statistic/ItStatisticTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/statistic/ItStatisticTest.java index 21cd93daf38..fd61a1a9a22 100644 --- a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/statistic/ItStatisticTest.java +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/statistic/ItStatisticTest.java @@ -49,13 +49,11 @@ public void testStatisticsRowCount() throws Exception { long prevValueOfThreshold = sqlStatisticManager.setThresholdTimeToPostponeUpdateMs(0); try { insertAndUpdateRunQuery(500); - // Minimum row count is 1000, even we have less rows. assertQuery(getUniqueQuery()) - .matches(scanRowCount("PUBLIC", "T", 1000)) + .matches(scanRowCount("PUBLIC", "T", 500)) .check(); insertAndUpdateRunQuery(600); - // Should return actual number of rows in the table. assertQuery(getUniqueQuery()) .matches(scanRowCount("PUBLIC", "T", 1100)) .check(); diff --git a/modules/sql-engine/src/integrationTest/sql/group1/join/test_not_distinct_from.test b/modules/sql-engine/src/integrationTest/sql/group1/join/test_not_distinct_from.test index 78ba34d3ca0..7fa55a31beb 100644 --- a/modules/sql-engine/src/integrationTest/sql/group1/join/test_not_distinct_from.test +++ b/modules/sql-engine/src/integrationTest/sql/group1/join/test_not_distinct_from.test @@ -45,7 +45,9 @@ NULL 2 #Vector with vector query III -select a.a, b.b, a.a IS NOT DISTINCT FROM b.b AS "Is Not Distinct From" FROM (VALUES (1), (2), (NULL)) AS a (a), (VALUES(1), (2), (NULL)) AS b (b) ORDER BY a.a NULLS LAST; +SELECT a.a, b.b, a.a IS NOT DISTINCT FROM b.b AS "Is Not Distinct From" + FROM (VALUES (1), (2), (NULL)) AS a (a), (VALUES(1), (2), (NULL)) AS b (b) + ORDER BY a.a NULLS LAST, b.b NULLS LAST; ---- 1 1 true 1 2 false @@ -58,7 +60,9 @@ NULL 2 false NULL NULL true query III -select a.a, b.b, a.a IS DISTINCT FROM b.b AS "Is Distinct From" FROM (VALUES (1), (2), (NULL)) AS a (a), (VALUES (1), (2), (NULL)) AS b (b) ORDER BY a.a NULLS LAST; +SELECT a.a, b.b, a.a IS DISTINCT FROM b.b AS "Is Distinct From" + FROM (VALUES (1), (2), (NULL)) AS a (a), (VALUES (1), (2), (NULL)) AS b (b) + ORDER BY a.a NULLS LAST, b.b NULLS LAST; ---- 1 1 false 1 2 true diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdRowCount.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdRowCount.java index 872625df653..462259731c4 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdRowCount.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdRowCount.java @@ -59,8 +59,8 @@ public class IgniteMdRowCount extends RelMdRowCount { /** {@inheritDoc} */ @Override - public Double getRowCount(Join rel, RelMetadataQuery mq) { - return rel.estimateRowCount(mq); + public @Nullable Double getRowCount(Join rel, RelMetadataQuery mq) { + return joinRowCount(mq, rel); } /** {@inheritDoc} */ diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java index 8b39e99bafe..b849757a8f1 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java @@ -18,13 +18,14 @@ package org.apache.ignite.internal.sql.engine.prepare; import static java.util.Objects.requireNonNull; -import static org.apache.ignite.internal.sql.engine.util.Commons.shortRuleName; +import static org.apache.ignite.internal.util.CollectionUtils.nullOrEmpty; import static org.apache.ignite.lang.ErrorGroups.Sql.STMT_PARSE_ERR; import java.io.PrintWriter; import java.io.Reader; import java.io.StringWriter; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -41,7 +42,6 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitDef; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.plan.volcano.VolcanoPlanner; @@ -51,6 +51,7 @@ import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.metadata.CachingRelMetadataProvider; @@ -70,6 +71,8 @@ import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlUtil; +import org.apache.calcite.sql.SqlWith; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.validate.SqlNonNullableAccessors; @@ -79,7 +82,6 @@ import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Planner; import org.apache.calcite.tools.Program; -import org.apache.calcite.tools.RuleSets; import org.apache.calcite.util.Pair; import org.apache.ignite.internal.logger.IgniteLogger; import org.apache.ignite.internal.logger.Loggers; @@ -235,6 +237,22 @@ public RelNode convert(SqlNode sql) { throw new UnsupportedOperationException(); } + /** Derives hints from given relation node, if possible. */ + List deriveHints(SqlNode node) { + SqlSelect select = null; + if (node instanceof SqlSelect) { + select = (SqlSelect) node; + } else if (node instanceof SqlWith && ((SqlWith) node).body instanceof SqlSelect) { + select = (SqlSelect) ((SqlWith) node).body; + } + + if (select != null && select.hasHints()) { + return SqlUtil.getRelHint(cluster.getHintStrategies(), select.getHints()); + } + + return List.of(); + } + /** * Preload some classes so that the time spent is not taken * into account when measuring query planning timeout. @@ -473,7 +491,7 @@ protected RelRoot trimUnusedFields(RelRoot root) { // prevent join-reordering. final SqlToRelConverter.Config config = sqlToRelConverterCfg .withExpand(false) - .withTrimUnusedFields(RelOptUtil.countJoins(root.rel) < 2); + .withTrimUnusedFields(true); SqlToRelConverter converter = sqlToRelConverter(validator(), catalogReader, config); boolean ordered = !root.collation.getFieldCollations().isEmpty(); boolean dml = SqlKind.DML.contains(root.kind); @@ -614,23 +632,26 @@ public IgniteSqlToRelConvertor sqlToRelConverter() { } /** - * Sets names of the rules which should be excluded from query optimization pipeline. + * Adds given rule to list of exclusion. * - * @param disabledRuleNames Names of the rules to exclude. The name can be derived from rule by - * {@link Commons#shortRuleName(RelOptRule)}. + * @param ruleName Name of the rule to exclude. The name can be derived from rule by {@link Commons#shortRuleName(RelOptRule)}. */ - public void setDisabledRules(Set disabledRuleNames) { - ctx.rulesFilter(rulesSet -> { - List newSet = new ArrayList<>(); + public void disableRule(String ruleName) { + ctx.disableRule(ruleName); + } - for (RelOptRule r : rulesSet) { - if (!disabledRuleNames.contains(shortRuleName(r))) { - newSet.add(r); - } - } + /** + * Adds all rules with name from given collections to list of exclusion. + * + * @param ruleNamesToDisable Names of the rules to exclude. The name can be derived from rule by + * {@link Commons#shortRuleName(RelOptRule)}. + */ + public void disableRules(Collection ruleNamesToDisable) { + if (nullOrEmpty(ruleNamesToDisable)) { + return; + } - return RuleSets.ofList(newSet); - }); + ruleNamesToDisable.forEach(this::disableRule); } private static class VolcanoPlannerExt extends VolcanoPlanner { diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java index de15dd1e572..c0564c33195 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java @@ -24,10 +24,10 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; @@ -35,8 +35,12 @@ import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.logical.LogicalCorrelate; +import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; +import org.apache.calcite.rel.rules.MultiJoin; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; @@ -49,6 +53,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlUtil; @@ -57,6 +62,8 @@ import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.util.ControlFlowException; import org.apache.calcite.util.Pair; +import org.apache.calcite.util.Util; +import org.apache.calcite.util.Util.FoundOne; import org.apache.ignite.internal.logger.IgniteLogger; import org.apache.ignite.internal.logger.Loggers; import org.apache.ignite.internal.sql.engine.hint.Hints; @@ -79,14 +86,17 @@ * Utility class that encapsulates the query optimization pipeline. */ public final class PlannerHelper { + static final RelOptRule JOIN_PUSH_THROUGH_JOIN_RULE = JoinPushThroughJoinRule.Config.RIGHT + .withOperandFor(LogicalJoin.class).toRule(); + /** - * Maximum number of tables in join supported for join order optimization. + * Maximum number of tables in join supported for exhaustive join order enumeration. * - *

If query joins more table than specified, then rules {@link CoreRules#JOIN_COMMUTE} and - * {@link CoreRules#JOIN_COMMUTE_OUTER} will be disabled, tables will be joined in the order - * of enumeration in the query. + *

Exhaustive join order enumeration implies that rules switching join inputs and rotating + * a tree of a two or more joins will be applied to every Join relation multiple times until + * they begin to produce variants which have been produced before. */ - private static final int MAX_SIZE_OF_JOIN_TO_OPTIMIZE = 5; + private static final int MAX_SIZE_OF_JOIN_FOR_EXHAUSTIVE_ENUMERATION = 5; private static final IgniteLogger LOG = Loggers.forClass(PlannerHelper.class); @@ -121,11 +131,11 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner) { RelNode rel = root.rel; - Hints hints = Hints.parse(root.hints); + Hints hints = Hints.parse(planner.deriveHints(sqlNode)); List disableRuleParams = hints.params(DISABLE_RULE); if (!disableRuleParams.isEmpty()) { - planner.setDisabledRules(Set.copyOf(disableRuleParams)); + planner.disableRules(disableRuleParams); } // Transformation chain @@ -137,17 +147,6 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner) { rel = planner.trimUnusedFields(root.withRel(rel)).rel; - boolean amountOfJoinsAreBig = hasTooMuchJoins(rel); - boolean enforceJoinOrder = hints.present(ENFORCE_JOIN_ORDER); - if (amountOfJoinsAreBig || enforceJoinOrder) { - Set disabledRules = new HashSet<>(disableRuleParams); - - disabledRules.add(shortRuleName(CoreRules.JOIN_COMMUTE)); - disabledRules.add(shortRuleName(CoreRules.JOIN_COMMUTE_OUTER)); - - planner.setDisabledRules(Set.copyOf(disabledRules)); - } - rel = planner.transform(PlannerPhase.HEP_FILTER_PUSH_DOWN, rel.getTraitSet(), rel); rel = planner.transform(PlannerPhase.HEP_PROJECT_PUSH_DOWN, rel.getTraitSet(), rel); @@ -164,6 +163,24 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner) { } } + boolean enforceJoinOrder = hints.present(ENFORCE_JOIN_ORDER); + boolean fallBackToExhaustiveJoinEnumeration = false; + if (!enforceJoinOrder) { + RelNode optimized = planner.transform(PlannerPhase.HEP_OPTIMIZE_JOIN_ORDER, rel.getTraitSet(), rel); + + if (hasMultiJoinNode(optimized)) { // HEP phase has failed optimization. + fallBackToExhaustiveJoinEnumeration = true; + } else { + rel = optimized; + } + } + + if (!fallBackToExhaustiveJoinEnumeration || hasTooMuchJoins(rel)) { + // All rules are enabled by default, but if we are happy with current join ordering, + // or number of relations is too big, we need to disable these rules to save optimization time. + planner.disableRules(exhaustiveJoinOrderingRules()); + } + RelTraitSet desired = rel.getCluster().traitSet() .replace(IgniteConvention.INSTANCE) .replace(IgniteDistributions.single()) @@ -234,10 +251,14 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner) { TableDescriptor descriptor = igniteTable.descriptor(); SqlBasicCall rowConstructor = (SqlBasicCall) rowConstructors.get(0); + SqlNodeList targetColumns = insertNode.getTargetColumnList(); + + // guaranteed by IgniteSqlValidator#validateInsert + assert targetColumns != null; Map columnToExpression = new HashMap<>(); for (int i = 0; i < rowConstructor.getOperandList().size(); i++) { - String columnName = ((SqlIdentifier) insertNode.getTargetColumnList().get(i)).getSimple(); + String columnName = ((SqlIdentifier) targetColumns.get(i)).getSimple(); SqlNode operand = rowConstructor.operand(i); if (operand.getKind() == SqlKind.DEFAULT) { @@ -286,7 +307,37 @@ private static boolean hasTooMuchJoins(RelNode rel) { joinSizeFinder.visit(rel); - return joinSizeFinder.sizeOfBiggestJoin() > MAX_SIZE_OF_JOIN_TO_OPTIMIZE; + return joinSizeFinder.sizeOfBiggestJoin() > MAX_SIZE_OF_JOIN_FOR_EXHAUSTIVE_ENUMERATION; + } + + private static boolean hasMultiJoinNode(RelNode root) { + try { + RelShuttle visitor = new RelHomogeneousShuttle() { + @Override + public RelNode visit(RelNode node) { + if (node instanceof MultiJoin) { + throw FoundOne.NULL; + } else { + return super.visit(node); + } + } + }; + + root.accept(visitor); + + return false; + } catch (Util.FoundOne ignored) { + return true; + } + } + + private static Set exhaustiveJoinOrderingRules() { + return Set.of( + // No need to add CoreRules.JOIN_COMMUTE_OUTER since it has the same short name + // as CoreRules.JOIN_COMMUTE, therefore it will be excluded as well + shortRuleName(CoreRules.JOIN_COMMUTE), + shortRuleName(JOIN_PUSH_THROUGH_JOIN_RULE) + ); } /** @@ -363,7 +414,7 @@ static boolean hasSubQuery(SqlNode node) { * @param node Query node. * @return Plan node with list of aliases, if the optimization is applicable. */ - public static @Nullable Pair> tryOptimizeSelectCount( + static @Nullable Pair> tryOptimizeSelectCount( IgnitePlanner planner, SqlNode node ) { diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerPhase.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerPhase.java index e362a3cabd4..c955e02e776 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerPhase.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerPhase.java @@ -19,11 +19,17 @@ import static org.apache.ignite.internal.sql.engine.prepare.IgnitePrograms.cbo; import static org.apache.ignite.internal.sql.engine.prepare.IgnitePrograms.hep; +import static org.apache.ignite.internal.sql.engine.prepare.PlannerHelper.JOIN_PUSH_THROUGH_JOIN_RULE; import java.util.ArrayList; import java.util.List; +import org.apache.calcite.plan.RelOptLattice; +import org.apache.calcite.plan.RelOptMaterialization; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelRule; +import org.apache.calcite.plan.hep.HepMatchOrder; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalFilter; @@ -35,8 +41,6 @@ import org.apache.calcite.rel.rules.FilterJoinRule.FilterIntoJoinRule; import org.apache.calcite.rel.rules.FilterMergeRule; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; -import org.apache.calcite.rel.rules.JoinPushExpressionsRule; -import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; import org.apache.calcite.rel.rules.ProjectFilterTransposeRule; import org.apache.calcite.rel.rules.ProjectMergeRule; import org.apache.calcite.rel.rules.ProjectRemoveRule; @@ -68,8 +72,10 @@ import org.apache.ignite.internal.sql.engine.rule.logical.ExposeIndexRule; import org.apache.ignite.internal.sql.engine.rule.logical.FilterScanMergeRule; import org.apache.ignite.internal.sql.engine.rule.logical.IgniteJoinConditionPushRule; +import org.apache.ignite.internal.sql.engine.rule.logical.IgniteMultiJoinOptimizeBushyRule; import org.apache.ignite.internal.sql.engine.rule.logical.LogicalOrToUnionRule; import org.apache.ignite.internal.sql.engine.rule.logical.ProjectScanMergeRule; +import org.apache.ignite.internal.sql.engine.util.Commons; /** * Represents a planner phase with its description and a used rule set. @@ -138,17 +144,49 @@ public Program getProgram(PlanningContext ctx) { } }, + HEP_OPTIMIZE_JOIN_ORDER( + "Heuristic phase to optimize join order" + ) { + @Override + public Program getProgram(PlanningContext ctx) { + return (planner, rel, traits, materializations, lattices) -> { + HepProgramBuilder builder = new HepProgramBuilder(); + + builder + .addSubprogram( + new HepProgramBuilder() + .addMatchOrder(HepMatchOrder.BOTTOM_UP) + .addRuleInstance(CoreRules.JOIN_TO_MULTI_JOIN) + .build() + ) + .addRuleInstance(IgniteMultiJoinOptimizeBushyRule.Config.DEFAULT.toRule()); + + HepPlanner hepPlanner = new HepPlanner(builder.build(), Commons.context(rel), true, + null, Commons.context(rel).config().getCostFactory()); + + hepPlanner.setExecutor(planner.getExecutor()); + + for (RelOptMaterialization materialization : materializations) { + hepPlanner.addMaterialization(materialization); + } + + for (RelOptLattice lattice : lattices) { + hepPlanner.addLattice(lattice); + } + + hepPlanner.setRoot(rel); + + return hepPlanner.findBestExp(); + }; + } + }, + OPTIMIZATION( "Main optimization phase", FilterMergeRule.Config.DEFAULT .withOperandFor(LogicalFilter.class).toRule(), - JoinPushThroughJoinRule.Config.RIGHT - .withOperandFor(LogicalJoin.class).toRule(), - - JoinPushExpressionsRule.Config.DEFAULT - .withOperandFor(LogicalJoin.class).toRule(), - + CoreRules.JOIN_PUSH_EXPRESSIONS, IgniteJoinConditionPushRule.INSTANCE, FilterIntoJoinRule.FilterIntoJoinRuleConfig.DEFAULT @@ -193,9 +231,11 @@ public Program getProgram(PlanningContext ctx) { CoreRules.MINUS_MERGE, CoreRules.INTERSECT_MERGE, CoreRules.UNION_REMOVE, - CoreRules.JOIN_COMMUTE, CoreRules.AGGREGATE_REMOVE, + + CoreRules.JOIN_COMMUTE, CoreRules.JOIN_COMMUTE_OUTER, + JOIN_PUSH_THROUGH_JOIN_RULE, PruneEmptyRules.CORRELATE_LEFT_INSTANCE, PruneEmptyRules.CORRELATE_RIGHT_INSTANCE, diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlanningContext.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlanningContext.java index 9a795ba187f..7822863c4f5 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlanningContext.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlanningContext.java @@ -20,23 +20,27 @@ import static org.apache.calcite.tools.Frameworks.createRootSchema; import static org.apache.ignite.internal.sql.engine.util.Commons.DISTRIBUTED_TRAITS_SET; import static org.apache.ignite.internal.sql.engine.util.Commons.FRAMEWORK_CONFIG; +import static org.apache.ignite.internal.sql.engine.util.Commons.shortRuleName; import com.google.common.collect.Multimap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMaps; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Properties; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.config.CalciteConnectionConfigImpl; import org.apache.calcite.config.CalciteConnectionProperty; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptSchema; import org.apache.calcite.plan.RelTraitDef; import org.apache.calcite.plan.volcano.VolcanoPlanner; @@ -61,6 +65,7 @@ import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.tools.RuleSet; +import org.apache.calcite.tools.RuleSets; import org.apache.calcite.util.CancelFlag; import org.apache.ignite.internal.sql.engine.metadata.cost.IgniteCostFactory; import org.apache.ignite.internal.sql.engine.rex.IgniteRexBuilder; @@ -152,7 +157,7 @@ public List> handlers(Class> hnd private final CancelFlag cancelFlag = new CancelFlag(new AtomicBoolean()); /** Rules which should be excluded for planning. */ - private Function rulesFilter; + private final Set rulesToDisable = new HashSet<>(); private IgnitePlanner planner; @@ -296,12 +301,23 @@ public static Builder builder() { /** Get rules filer. */ public RuleSet rules(RuleSet set) { - return rulesFilter != null ? rulesFilter.apply(set) : set; + if (rulesToDisable.isEmpty()) { + return set; + } + + List filtered = new ArrayList<>(); + for (RelOptRule r : set) { + if (!rulesToDisable.contains(shortRuleName(r))) { + filtered.add(r); + } + } + + return RuleSets.ofList(filtered); } - /** Set rules filter. */ - public void rulesFilter(Function rulesFilter) { - this.rulesFilter = rulesFilter; + /** Add rule with given name to list of exclusions. */ + public void disableRule(String ruleName) { + rulesToDisable.add(ruleName); } /** Set a flag indicating that the planning was canceled due to a timeout. */ diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/HashJoinConverterRule.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/HashJoinConverterRule.java index be9218d5ac2..30b32009f48 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/HashJoinConverterRule.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/HashJoinConverterRule.java @@ -35,6 +35,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.ignite.internal.sql.engine.rel.IgniteConvention; import org.apache.ignite.internal.sql.engine.rel.IgniteHashJoin; +import org.apache.ignite.internal.sql.engine.trait.IgniteDistributions; /** * Hash join converter. @@ -85,12 +86,11 @@ public boolean matches(RelOptRuleCall call) { @Override protected PhysicalNode convert(RelOptPlanner planner, RelMetadataQuery mq, LogicalJoin rel) { RelOptCluster cluster = rel.getCluster(); - RelTraitSet outTraits = cluster.traitSetOf(IgniteConvention.INSTANCE); - RelTraitSet leftInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE); - RelTraitSet rightInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE); - RelNode left = convert(rel.getLeft(), leftInTraits); - RelNode right = convert(rel.getRight(), rightInTraits); + RelTraitSet traits = cluster.traitSetOf(IgniteConvention.INSTANCE) + .replace(IgniteDistributions.single()); + RelNode left = convert(rel.getLeft(), traits); + RelNode right = convert(rel.getRight(), traits); - return new IgniteHashJoin(cluster, outTraits, left, right, rel.getCondition(), rel.getVariablesSet(), rel.getJoinType()); + return new IgniteHashJoin(cluster, traits, left, right, rel.getCondition(), rel.getVariablesSet(), rel.getJoinType()); } } diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/MergeJoinConverterRule.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/MergeJoinConverterRule.java index 72b60eb258a..33fdc231ed1 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/MergeJoinConverterRule.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/MergeJoinConverterRule.java @@ -32,6 +32,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.ignite.internal.sql.engine.rel.IgniteConvention; import org.apache.ignite.internal.sql.engine.rel.IgniteMergeJoin; +import org.apache.ignite.internal.sql.engine.trait.IgniteDistributions; /** * Ignite Join converter. @@ -63,9 +64,12 @@ protected PhysicalNode convert(RelOptPlanner planner, RelMetadataQuery mq, Logic JoinInfo joinInfo = JoinInfo.of(rel.getLeft(), rel.getRight(), rel.getCondition()); RelTraitSet leftInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE) + .replace(IgniteDistributions.single()) .replace(RelCollations.of(joinInfo.leftKeys)); - RelTraitSet outTraits = cluster.traitSetOf(IgniteConvention.INSTANCE); + RelTraitSet outTraits = cluster.traitSetOf(IgniteConvention.INSTANCE) + .replace(IgniteDistributions.single()); RelTraitSet rightInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE) + .replace(IgniteDistributions.single()) .replace(RelCollations.of(joinInfo.rightKeys)); RelNode left = convert(rel.getLeft(), leftInTraits); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/NestedLoopJoinConverterRule.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/NestedLoopJoinConverterRule.java index 44f8783813d..0c216bc8825 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/NestedLoopJoinConverterRule.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/NestedLoopJoinConverterRule.java @@ -27,6 +27,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.ignite.internal.sql.engine.rel.IgniteConvention; import org.apache.ignite.internal.sql.engine.rel.IgniteNestedLoopJoin; +import org.apache.ignite.internal.sql.engine.trait.IgniteDistributions; /** * Ignite Join converter. @@ -45,12 +46,11 @@ public NestedLoopJoinConverterRule() { @Override protected PhysicalNode convert(RelOptPlanner planner, RelMetadataQuery mq, LogicalJoin rel) { RelOptCluster cluster = rel.getCluster(); - RelTraitSet outTraits = cluster.traitSetOf(IgniteConvention.INSTANCE); - RelTraitSet leftInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE); - RelTraitSet rightInTraits = cluster.traitSetOf(IgniteConvention.INSTANCE); - RelNode left = convert(rel.getLeft(), leftInTraits); - RelNode right = convert(rel.getRight(), rightInTraits); + RelTraitSet traits = cluster.traitSetOf(IgniteConvention.INSTANCE) + .replace(IgniteDistributions.single()); + RelNode left = convert(rel.getLeft(), traits); + RelNode right = convert(rel.getRight(), traits); - return new IgniteNestedLoopJoin(cluster, outTraits, left, right, rel.getCondition(), rel.getVariablesSet(), rel.getJoinType()); + return new IgniteNestedLoopJoin(cluster, traits, left, right, rel.getCondition(), rel.getVariablesSet(), rel.getJoinType()); } } diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/IgniteMultiJoinOptimizeBushyRule.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/IgniteMultiJoinOptimizeBushyRule.java new file mode 100644 index 00000000000..a071fd8cd7c --- /dev/null +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rule/logical/IgniteMultiJoinOptimizeBushyRule.java @@ -0,0 +1,456 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.rule.logical; + +import static java.lang.Integer.bitCount; +import static org.apache.ignite.internal.util.IgniteUtils.isPow2; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.Comparator; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.rules.LoptMultiJoin; +import org.apache.calcite.rel.rules.MultiJoin; +import org.apache.calcite.rel.rules.TransformationRule; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexPermuteInputsShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.mapping.Mappings; +import org.apache.calcite.util.mapping.Mappings.TargetMapping; +import org.immutables.value.Value; +import org.jetbrains.annotations.Nullable; + +/** + * Transformation rule used for optimizing multi-join queries using a bushy join tree strategy. + * + *

This is an implementation of subset-driven enumeration algorithm (Inspired by G. Moerkotte and T. Neumann. + * Analysis of Two Existing and One New Dynamic Programming Algorithm for the Generation of Optimal Bushy Join + * Trees without Cross Products. 2.2 Subset-Driven Enumeration). The main loop enumerates all subsets of relations + * in a way suitable for dynamic programming: it guarantees, that for every emitted set {@code S}, any split of the + * {@code S} will produce subsets which have been already processed: + * 

+ *     For example, for join of 4 relations it will produce following sequence: 
+ *         0011
+ *         0101
+ *         0110
+ *         ...
+ *         1101
+ *         1110
+ *         1111
+ *
+ * + *

The inner while-loop enumerates all possible splits of given subset {@code S} on disjoint subset + * {@code lhs} and {@code rhs} such that {@code lhs ∪ rhs = S} (Inspired by B. Vance and D. Maier. + * Rapid bushy join-order optimization with cartesian products). + * + *

Finally, if the initial set of relations is not connected, the algorithm composes cartesian join + * from best plans, until all relations are joined. + * + *

Current limitations are as follow:

    + *
  1. Only INNER joins are supported
    2. + *
  2. Number of relations to optimize is limited to 20. This is due to time and memory complexity of algorithm chosen.
    3. + *
  3. Disjunctive predicate is not considered as connections.
    4. + *
+ */ +@Value.Enclosing +public class IgniteMultiJoinOptimizeBushyRule + extends RelRule + implements TransformationRule { + + private static final int MAX_JOIN_SIZE = 20; + + /** + * Comparator that puts better vertexes first. + * + *

Better vertex is the one that incorporate more relations, or costs less. + */ + private static final Comparator VERTEX_COMPARATOR = + Comparator.comparingInt(v -> v.size) + .reversed() + .thenComparingDouble(v -> v.cost); + + /** Creates a MultiJoinOptimizeBushyRule. */ + private IgniteMultiJoinOptimizeBushyRule(Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall call) { + MultiJoin multiJoinRel = call.rel(0); + + int numberOfRelations = multiJoinRel.getInputs().size(); + if (numberOfRelations > MAX_JOIN_SIZE) { + return; + } + + // Currently, algorithm below can handle only INNER JOINs + if (multiJoinRel.isFullOuterJoin()) { + return; + } + + for (JoinRelType joinType : multiJoinRel.getJoinTypes()) { + if (joinType != JoinRelType.INNER) { + return; + } + } + + LoptMultiJoin multiJoin = new LoptMultiJoin(multiJoinRel); + + RexBuilder rexBuilder = multiJoinRel.getCluster().getRexBuilder(); + RelBuilder relBuilder = call.builder(); + RelMetadataQuery mq = call.getMetadataQuery(); + + List unusedConditions = new ArrayList<>(); + + Int2ObjectMap> edges = collectEdges(multiJoin, unusedConditions); + Int2ObjectMap bestPlan = new Int2ObjectOpenHashMap<>(); + BitSet connections = new BitSet(1 << numberOfRelations); + + int id = 0b1; + int fieldOffset = 0; + for (RelNode input : multiJoinRel.getInputs()) { + TargetMapping mapping = Mappings.offsetSource( + Mappings.createIdentity(input.getRowType().getFieldCount()), + fieldOffset, + multiJoin.getNumTotalFields() + ); + + bestPlan.put(id, new Vertex(id, mq.getRowCount(input), input, mapping)); + connections.set(id); + + id <<= 1; + fieldOffset += input.getRowType().getFieldCount(); + } + + Vertex bestSoFar = null; + for (int s = 0b11; s < 1 << numberOfRelations; s++) { + if (isPow2(s)) { + // Single relations have been processed during initialization. + continue; + } + + int lhs = Integer.lowestOneBit(s); + while (lhs < (s / 2) + 1) { + int rhs = s - lhs; + + List edges0; + if (connections.get(lhs) && connections.get(rhs)) { + edges0 = findEdges(lhs, rhs, edges); + } else { + edges0 = List.of(); + } + + if (!edges0.isEmpty()) { + connections.set(s); + + Vertex planLhs = bestPlan.get(lhs); + Vertex planRhs = bestPlan.get(rhs); + + Vertex newPlan = createJoin(planLhs, planRhs, edges0, mq, relBuilder, rexBuilder); + Vertex currentBest = bestPlan.get(s); + if (currentBest == null || currentBest.cost > newPlan.cost) { + bestPlan.put(s, newPlan); + + bestSoFar = chooseBest(bestSoFar, newPlan); + } + + aggregateEdges(edges, lhs, rhs); + } + + lhs = s & (lhs - s); + } + } + + int allRelationsMask = (1 << numberOfRelations) - 1; + + Vertex best; + if (bestSoFar == null || bestSoFar.id != allRelationsMask) { + best = composeCartesianJoin(allRelationsMask, bestPlan, edges, bestSoFar, mq, relBuilder, rexBuilder); + } else { + best = bestSoFar; + } + + RelNode result = relBuilder + .push(best.rel) + .filter(RexUtil.composeConjunction(rexBuilder, unusedConditions) + .accept(new RexPermuteInputsShuttle(best.mapping, best.rel))) + .project(relBuilder.fields(best.mapping)) + .build(); + + call.transformTo(result); + } + + private static void aggregateEdges(Int2ObjectMap> edges, int lhs, int rhs) { + int id = lhs | rhs; + if (!edges.containsKey(id)) { + Set used = Collections.newSetFromMap(new IdentityHashMap<>()); + + List union = new ArrayList<>(edges.getOrDefault(lhs, List.of())); + used.addAll(union); + + edges.getOrDefault(rhs, List.of()).forEach(edge -> { + if (used.add(edge)) { + union.add(edge); + } + }); + + if (!union.isEmpty()) { + edges.put(id, union); + } + } + } + + private static Vertex composeCartesianJoin( + int allRelationsMask, + Int2ObjectMap bestPlan, + Int2ObjectMap> edges, + @Nullable Vertex bestSoFar, + RelMetadataQuery mq, + RelBuilder relBuilder, + RexBuilder rexBuilder + ) { + List options; + + if (bestSoFar != null) { + options = new ArrayList<>(); + + for (Vertex option : bestPlan.values()) { + if ((option.id & bestSoFar.id) == 0) { + options.add(option); + } + } + } else { + options = new ArrayList<>(bestPlan.values()); + } + + options.sort(VERTEX_COMPARATOR); + + Iterator it = options.iterator(); + + if (bestSoFar == null) { + bestSoFar = it.next(); + } + + while (it.hasNext() && bestSoFar.id != allRelationsMask) { + Vertex input = it.next(); + + if ((bestSoFar.id & input.id) != 0) { + continue; + } + + List edges0 = findEdges(bestSoFar.id, input.id, edges); + + aggregateEdges(edges, bestSoFar.id, input.id); + + bestSoFar = createJoin(bestSoFar, input, edges0, mq, relBuilder, rexBuilder); + } + + assert bestSoFar.id == allRelationsMask; + + return bestSoFar; + } + + private static Vertex chooseBest(@Nullable Vertex currentBest, Vertex candidate) { + if (currentBest == null) { + return candidate; + } + + if (VERTEX_COMPARATOR.compare(currentBest, candidate) > 0) { + return candidate; + } + + return currentBest; + } + + private static Int2ObjectMap> collectEdges(LoptMultiJoin multiJoin, List unusedConditions) { + Int2ObjectMap> edges = new Int2ObjectOpenHashMap<>(); + + for (RexNode condition : multiJoin.getJoinFilters()) { + int[] inputRefs = multiJoin.getFactorsRefByJoinFilter(condition).toArray(); + + // No need to collect conditions involving a single table, because 1) during main loop + // we will be looking only for edges connecting two subsets, and condition referring to + // a single table never meet this condition, and 2) for inner join such conditions must + // be pushed down already, and we rely on this fact. + if (inputRefs.length < 2) { + unusedConditions.add(condition); + continue; + } + + // TODO: https://issues.apache.org/jira/browse/IGNITE-24210 the whole if-block need to be removed + if (condition.isA(SqlKind.OR)) { + unusedConditions.add(condition); + continue; + } + + int connectedInputs = 0; + for (int i : inputRefs) { + connectedInputs |= 1 << i; + } + + Edge edge = new Edge(connectedInputs, condition); + for (int i : inputRefs) { + edges.computeIfAbsent(1 << i, k -> new ArrayList<>()).add(edge); + } + } + + return edges; + } + + private static Vertex createJoin( + Vertex lhs, + Vertex rhs, + List edges, + RelMetadataQuery metadataQuery, + RelBuilder relBuilder, + RexBuilder rexBuilder + ) { + List conditions = new ArrayList<>(); + + for (Edge e : edges) { + conditions.add(e.condition); + } + + double leftSize = metadataQuery.getRowCount(lhs.rel); + double rightSize = metadataQuery.getRowCount(rhs.rel); + + Vertex majorFactor; + Vertex minorFactor; + + // Let's put bigger input on left side, because right side will probably be materialized. + if (leftSize >= rightSize) { + majorFactor = lhs; + minorFactor = rhs; + } else { + majorFactor = rhs; + minorFactor = lhs; + } + + TargetMapping mapping = Mappings.merge( + majorFactor.mapping, + Mappings.offsetTarget( + minorFactor.mapping, + majorFactor.rel.getRowType().getFieldCount() + ) + ); + + RexNode condition = RexUtil.composeConjunction(rexBuilder, conditions) + .accept(new RexPermuteInputsShuttle(mapping, majorFactor.rel, minorFactor.rel)); + + RelNode join = relBuilder + .push(majorFactor.rel) + .push(minorFactor.rel) + .join(JoinRelType.INNER, condition) + .build(); + + double selfCost = metadataQuery.getRowCount(join); + + return new Vertex(lhs.id | rhs.id, selfCost + lhs.cost + rhs.cost, join, mapping); + } + + /** + * Finds all edges which connect given subsets. + * + *

Returned edges will satisfy two conditions:

    + *
  1. At least one relation from each side will be covered by edge.
    2. + *
  2. No any other relations outside of {@code lhs ∪ rhs} will be covered by edge.
    3. + *
+ * + * @param lhs Left subset. + * @param rhs Right subset. + * @param edges All edges. + * @return List of edges connecting given subsets. + */ + private static List findEdges( + int lhs, + int rhs, + Int2ObjectMap> edges + ) { + List result = new ArrayList<>(); + List fromLeft = edges.getOrDefault(lhs, List.of()); + for (Edge edge : fromLeft) { + int requiredInputs = edge.connectedInputs & ~lhs; + if (requiredInputs == 0 || edge.connectedInputs == requiredInputs) { + continue; + } + + requiredInputs &= ~rhs; + if (requiredInputs == 0) { + result.add(edge); + } + } + + return result; + } + + private static class Edge { + /** Bitmap of all inputs connected by condition. */ + private final int connectedInputs; + private final RexNode condition; + + Edge(int connectedInputs, RexNode condition) { + this.connectedInputs = connectedInputs; + this.condition = condition; + } + } + + private static class Vertex { + /** Bitmap of inputs joined together so far with current vertex served as a root. */ + private final int id; + /** Number of inputs joined together so far with current vertex served as a root. */ + private final byte size; + /** Cumulative cost of the tree. */ + private final double cost; + private final TargetMapping mapping; + private final RelNode rel; + + Vertex(int id, double cost, RelNode rel, TargetMapping mapping) { + this.id = id; + this.size = (byte) bitCount(id); + this.cost = cost; + this.rel = rel; + this.mapping = mapping; + } + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableIgniteMultiJoinOptimizeBushyRule.Config.of() + .withOperandSupplier(b -> b.operand(MultiJoin.class).anyInputs()); + + @Override + default IgniteMultiJoinOptimizeBushyRule toRule() { + return new IgniteMultiJoinOptimizeBushyRule(this); + } + } +} diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImpl.java index 1148eeb0d25..f70495a18f2 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImpl.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImpl.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.sql.engine.statistic; import static org.apache.ignite.internal.event.EventListener.fromConsumer; +import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture; import java.util.Collection; import java.util.List; @@ -26,6 +27,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.internal.catalog.CatalogService; import org.apache.ignite.internal.catalog.descriptors.CatalogTableDescriptor; import org.apache.ignite.internal.catalog.events.CatalogEvent; @@ -49,14 +51,13 @@ public class SqlStatisticManagerImpl implements SqlStatisticManager { private static final IgniteLogger LOG = Loggers.forClass(SqlStatisticManagerImpl.class); private static final long DEFAULT_TABLE_SIZE = 1_000_000L; - private static final long MINIMUM_TABLE_SIZE = 1_000L; private static final ActualSize DEFAULT_VALUE = new ActualSize(DEFAULT_TABLE_SIZE, 0L); private final EventListener lwmListener = fromConsumer(this::onLwmChanged); private final EventListener dropTableEventListener = fromConsumer(this::onTableDrop); private final EventListener createTableEventListener = fromConsumer(this::onTableCreate); - private volatile Future statisticUpdateFut = CompletableFuture.completedFuture(null); + private final AtomicReference> latestUpdateFut = new AtomicReference<>(nullCompletedFuture()); /** A queue for deferred table destruction events. */ private final LongPriorityQueue destructionEventsQueue = new LongPriorityQueue<>(DestroyTableEvent::catalogVersion); @@ -88,14 +89,13 @@ public SqlStatisticManagerImpl(TableManager tableManager, CatalogService catalog */ @Override public long tableSize(int tableId) { - updateTableSizeStatistics(tableId); - long tableSize = tableSizeMap.getOrDefault(tableId, DEFAULT_VALUE).getSize(); + updateTableSizeStatistics(tableId, false); - return Math.max(tableSize, MINIMUM_TABLE_SIZE); + return tableSizeMap.getOrDefault(tableId, DEFAULT_VALUE).getSize(); } /** Update table size statistic in the background if it required. */ - private void updateTableSizeStatistics(int tableId) { + private void updateTableSizeStatistics(int tableId, boolean force) { TableViewInternal tableView = tableManager.cachedTable(tableId); if (tableView == null) { LOG.debug("There is no table to update statistics [id={}].", tableId); @@ -110,14 +110,14 @@ private void updateTableSizeStatistics(int tableId) { long currTimestamp = FastTimestamps.coarseCurrentTimeMillis(); long lastUpdateTime = tableSize.getTimestamp(); - if (lastUpdateTime <= currTimestamp - thresholdTimeToPostponeUpdateMs) { + if (force || lastUpdateTime <= currTimestamp - thresholdTimeToPostponeUpdateMs) { // Prevent to run update for the same table twice concurrently. - if (!tableSizeMap.replace(tableId, tableSize, new ActualSize(tableSize.getSize(), currTimestamp))) { + if (!force && !tableSizeMap.replace(tableId, tableSize, new ActualSize(tableSize.getSize(), currTimestamp))) { return; } // just request new table size in background. - statisticUpdateFut = tableView.internalTable().estimatedSize() + CompletableFuture updateResult = tableView.internalTable().estimatedSize() .thenAccept(size -> { // the table can be concurrently dropped and we shouldn't put new value in this case. tableSizeMap.computeIfPresent(tableId, (k, v) -> new ActualSize(size, currTimestamp)); @@ -125,6 +125,8 @@ private void updateTableSizeStatistics(int tableId) { LOG.info("Can't calculate size for table [id={}].", e, tableId); return null; }); + + latestUpdateFut.updateAndGet(prev -> prev == null ? updateResult : prev.thenCompose(none -> updateResult)); } } @@ -225,6 +227,16 @@ public long setThresholdTimeToPostponeUpdateMs(long milliseconds) { */ @TestOnly public Future lastUpdateStatisticFuture() { - return statisticUpdateFut; + return latestUpdateFut.get(); + } + + /** Forcibly updates statistics for all known tables, ignoring throttling. */ + @TestOnly + public void forceUpdateAll() { + List tableIds = List.copyOf(tableSizeMap.keySet()); + + for (int tableId : tableIds) { + updateTableSizeStatistics(tableId, true); + } } } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java index 4645b0d9f43..6f7b090c30c 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java @@ -21,7 +21,6 @@ import static java.util.stream.Collectors.toCollection; import static org.apache.ignite.internal.lang.IgniteStringFormatter.format; import static org.apache.ignite.internal.sql.engine.exec.ExecutionServiceImplTest.PLANNING_THREAD_COUNT; -import static org.apache.ignite.internal.sql.engine.exec.ExecutionServiceImplTest.PLANNING_TIMEOUT; import static org.apache.ignite.internal.testframework.IgniteTestUtils.await; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; import static org.apache.ignite.internal.util.CollectionUtils.nullOrEmpty; @@ -405,6 +404,15 @@ public interface ClusterBuilder { * @return {@code this} for chaining. */ ClusterBuilder registerSystemView(String nodeName, String systemViewName); + + /** + * Sets a timeout for query optimization phase. + * + * @param value A planning timeout value. + * @param timeUnit A time unit. + * @return {@code this} for chaining. + */ + ClusterBuilder planningTimeout(long value, TimeUnit timeUnit); } /** @@ -616,6 +624,7 @@ private static class ClusterBuilderImpl implements ClusterBuilder { private final List> systemViews = new ArrayList<>(); private final Map> nodeName2SystemView = new HashMap<>(); + private long planningTimeout = TimeUnit.SECONDS.toMillis(15); private @Nullable DefaultDataProvider defaultDataProvider = null; private @Nullable DefaultAssignmentsProvider defaultAssignmentsProvider = null; @@ -663,6 +672,13 @@ public ClusterBuilder defaultAssignmentsProvider(DefaultAssignmentsProvider defa return this; } + @Override + public ClusterBuilder planningTimeout(long value, TimeUnit timeUnit) { + this.planningTimeout = timeUnit.toMillis(value); + + return this; + } + /** {@inheritDoc} */ @Override public TestCluster build() { @@ -678,7 +694,7 @@ public TestCluster build() { ConcurrentMap tablesSize = new ConcurrentHashMap<>(); var schemaManager = createSqlSchemaManager(catalogManager, tablesSize); var prepareService = new PrepareServiceImpl(clusterName, 0, CaffeineCacheFactory.INSTANCE, - new DdlSqlToCommandConverter(), PLANNING_TIMEOUT, PLANNING_THREAD_COUNT, + new DdlSqlToCommandConverter(), planningTimeout, PLANNING_THREAD_COUNT, new NoOpMetricManager(), schemaManager); Map> systemViewsByNode = new HashMap<>(); diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java index 4a719ee9879..da93d2b79e3 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java @@ -288,7 +288,7 @@ protected PlanningContext plannerCtx( assertNotNull(planner); - planner.setDisabledRules(Set.copyOf(Arrays.asList(disabledRules))); + planner.disableRules(Arrays.asList(disabledRules)); return ctx; } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractTpcQueryPlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractTpcQueryPlannerTest.java new file mode 100644 index 00000000000..601396aa42f --- /dev/null +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractTpcQueryPlannerTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.planner; + +import static java.lang.annotation.ElementType.TYPE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static org.apache.ignite.internal.sql.engine.util.Commons.cast; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.common.io.CharStreams; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.ignite.internal.sql.engine.framework.TestBuilders; +import org.apache.ignite.internal.sql.engine.framework.TestCluster; +import org.apache.ignite.internal.sql.engine.framework.TestNode; +import org.apache.ignite.internal.sql.engine.prepare.MultiStepPlan; +import org.apache.ignite.internal.sql.engine.util.Cloner; +import org.apache.ignite.internal.sql.engine.util.Commons; +import org.apache.ignite.internal.sql.engine.util.TpcScaleFactor; +import org.apache.ignite.internal.sql.engine.util.TpcTable; +import org.apache.ignite.internal.sql.engine.util.tpch.TpchHelper; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInfo; + +/** + * Abstract test class to ensure a planner generates optimal plan for TPC queries. + * + *

Any derived class must be annotated with {@link TpcSuiteInfo}. + */ +abstract class AbstractTpcQueryPlannerTest extends AbstractPlannerTest { + private static TestCluster CLUSTER; + + private static Function queryLoader; + private static Function planLoader; + + @BeforeAll + static void startCluster(TestInfo info) throws NoSuchMethodException { + Class testClass = info.getTestClass().orElseThrow(); + + TpcSuiteInfo suiteInfo = testClass.getAnnotation(TpcSuiteInfo.class); + + if (suiteInfo == null) { + throw new IllegalStateException("Class " + testClass + " must be annotated with @" + TpcSuiteInfo.class.getSimpleName()); + } + + Method queryLoaderMethod = testClass.getDeclaredMethod(suiteInfo.queryLoader(), String.class); + Method planLoaderMethod = testClass.getDeclaredMethod(suiteInfo.planLoader(), String.class); + + queryLoader = queryId -> invoke(queryLoaderMethod, queryId); + planLoader = queryId -> invoke(planLoaderMethod, queryId); + + CLUSTER = TestBuilders.cluster() + .nodes("N1") + .planningTimeout(1, TimeUnit.MINUTES) + .build(); + CLUSTER.start(); + + TestNode node = CLUSTER.node("N1"); + + TpcTable[] tables = cast(suiteInfo.tables().getEnumConstants()); + for (TpcTable table : tables) { + node.initSchema(table.ddlScript()); + + CLUSTER.setTableSize(table.tableName().toUpperCase(), table.estimatedSize(TpcScaleFactor.SF_1GB)); + } + } + + @AfterAll + static void stopCluster() throws Exception { + CLUSTER.stop(); + } + + static void validateQueryPlan(String queryId) { + TestNode node = CLUSTER.node("N1"); + + MultiStepPlan plan = (MultiStepPlan) node.prepare(queryLoader.apply(queryId)); + + String actualPlan = RelOptUtil.toString(Cloner.clone(plan.root(), Commons.cluster()), SqlExplainLevel.DIGEST_ATTRIBUTES); + String expectedPlan = planLoader.apply(queryId); + + assertEquals(expectedPlan, actualPlan); + } + + static String loadFromResource(String resource) { + try (InputStream is = TpchHelper.class.getClassLoader().getResourceAsStream(resource)) { + if (is == null) { + throw new IllegalArgumentException("Resource does not exist: " + resource); + } + try (InputStreamReader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) { + return CharStreams.toString(reader); + } + } catch (IOException e) { + throw new UncheckedIOException("I/O operation failed: " + resource, e); + } + } + + private static T invoke(Method method, Object... arguments) { + try { + return (T) method.invoke(null, arguments); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + @Target(TYPE) + @Retention(RUNTIME) + public @interface TpcSuiteInfo { + /** Returns enum representing set of tables to initialize for test. */ + Class> tables(); + + /** + * Returns name of the method to use as query loader. + * + *

Specified method must be static method within class this annotation is specified upon. + * Specified method must accept a single parameter of a string type which is query id, and return + * string representing a query text. + */ + String queryLoader(); + + /** + * Returns name of the method to use as plan loader. + * + *

Specified method must be static method within class this annotation is specified upon. + * Specified method must accept a single parameter of a string type which is query id, and return + * string representing a query plan. + */ + String planLoader(); + } +} diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/JoinCommutePlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/JoinCommutePlannerTest.java deleted file mode 100644 index d394d465bab..00000000000 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/JoinCommutePlannerTest.java +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.sql.engine.planner; - -import static org.apache.ignite.internal.lang.IgniteStringFormatter.format; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.calcite.plan.RelOptCost; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.rules.CoreRules; -import org.apache.calcite.rel.rules.FilterJoinRule.JoinConditionPushRule; -import org.apache.calcite.rel.rules.FilterJoinRule.JoinConditionPushRule.JoinConditionPushRuleConfig; -import org.apache.calcite.rel.rules.JoinCommuteRule; -import org.apache.ignite.internal.sql.engine.framework.TestBuilders; -import org.apache.ignite.internal.sql.engine.prepare.PlanningContext; -import org.apache.ignite.internal.sql.engine.rel.IgniteNestedLoopJoin; -import org.apache.ignite.internal.sql.engine.rel.IgniteProject; -import org.apache.ignite.internal.sql.engine.rel.IgniteRel; -import org.apache.ignite.internal.sql.engine.rel.IgniteTableScan; -import org.apache.ignite.internal.sql.engine.schema.IgniteSchema; -import org.apache.ignite.internal.sql.engine.schema.IgniteTable; -import org.apache.ignite.internal.sql.engine.trait.IgniteDistributions; -import org.apache.ignite.internal.type.NativeTypes; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -/** - * Tests correctness applying of JOIN_COMMUTE* rules. - */ -public class JoinCommutePlannerTest extends AbstractPlannerTest { - private static IgniteSchema publicSchema; - - /** - * Set up tests. - */ - @BeforeAll - public static void init() { - publicSchema = createSchema( - TestBuilders.table() - .name("HUGE") - .addColumn("ID", NativeTypes.INT32) - .distribution(IgniteDistributions.affinity(0, nextTableId(), DEFAULT_ZONE_ID)) - .size(1_000) - .build(), - TestBuilders.table() - .name("SMALL") - .addColumn("ID", NativeTypes.INT32) - .distribution(IgniteDistributions.affinity(0, nextTableId(), DEFAULT_ZONE_ID)) - .size(10) - .build() - ); - } - - @Test - @Disabled("https://issues.apache.org/jira/browse/IGNITE-16334") - public void testEnforceJoinOrderHint() throws Exception { - String sqlJoinCommuteWithNoHint = "SELECT COUNT(*) FROM SMALL s, HUGE h, HUGE h1 WHERE h.id = s.id and h1.id=s.id"; - String sqlJoinCommuteWithHint = - "SELECT /*+ ENFORCE_JOIN_ORDER */ COUNT(*) FROM SMALL s, HUGE h, HUGE h1 WHERE h.id = s.id and h1.id=s.id"; - String sqlJoinCommuteOuterWithNoHint = "SELECT COUNT(*) FROM SMALL s RIGHT JOIN HUGE h on h.id = s.id"; - String sqlJoinCommuteOuterWithHint = "SELECT /*+ ENFORCE_JOIN_ORDER */ COUNT(*) FROM SMALL s RIGHT JOIN HUGE h on h.id = s.id"; - - JoinConditionPushRule joinConditionPushRule = JoinConditionPushRuleConfig.DEFAULT.toRule(); - JoinCommuteRule joinCommuteRule = JoinCommuteRule.Config.DEFAULT.toRule(); - - RuleAttemptListener listener = new RuleAttemptListener(); - - physicalPlan(sqlJoinCommuteWithNoHint, publicSchema, listener); - assertTrue(listener.isApplied(joinCommuteRule)); - assertTrue(listener.isApplied(joinConditionPushRule)); - - listener.reset(); - physicalPlan(sqlJoinCommuteOuterWithNoHint, publicSchema, listener); - assertTrue(listener.isApplied(joinCommuteRule)); - assertTrue(listener.isApplied(joinConditionPushRule)); - - listener.reset(); - physicalPlan(sqlJoinCommuteWithHint, publicSchema, listener); - assertFalse(listener.isApplied(joinCommuteRule)); - assertTrue(listener.isApplied(joinConditionPushRule)); - - listener.reset(); - physicalPlan(sqlJoinCommuteOuterWithHint, publicSchema, listener); - assertFalse(listener.isApplied(joinCommuteRule)); - assertTrue(listener.isApplied(joinConditionPushRule)); - } - - @Test - public void testOuterCommute() throws Exception { - // Use aggregates that are the same for both MAP and REDUCE phases. - String sql = "SELECT SUM(s.id), SUM(h.id) FROM SMALL s RIGHT JOIN HUGE h on h.id = s.id"; - - IgniteRel phys = physicalPlan(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin"); - - assertNotNull(phys); - - IgniteNestedLoopJoin join = findFirstNode(phys, byClass(IgniteNestedLoopJoin.class)); - - IgniteProject proj = findFirstNode(phys, byClass(IgniteProject.class)); - - assertNotNull(proj); - - assertEquals(JoinRelType.LEFT, join.getJoinType()); - - PlanningContext ctx = plannerCtx(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin"); - - RelOptPlanner pl = ctx.cluster().getPlanner(); - - RelOptCost costWithCommute = pl.getCost(phys, phys.getCluster().getMetadataQuery()); - - assertNotNull(costWithCommute); - - System.out.println("plan: " + RelOptUtil.toString(phys)); - - assertNotNull(phys); - - phys = physicalPlan(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin", "JoinCommuteRule"); - - join = findFirstNode(phys, byClass(IgniteNestedLoopJoin.class)); - - proj = findFirstNode(phys, byClass(IgniteProject.class)); - - assertNull(proj); - - // no commute - assertEquals(JoinRelType.RIGHT, join.getJoinType()); - - ctx = plannerCtx(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin", "JoinCommuteRule"); - - pl = ctx.cluster().getPlanner(); - - RelOptCost costWithoutCommute = pl.getCost(phys, phys.getCluster().getMetadataQuery()); - - System.out.println("plan: " + RelOptUtil.toString(phys)); - - assertTrue(costWithCommute.isLt(costWithoutCommute)); - } - - @Test - public void testInnerCommute() throws Exception { - // Use aggregates that are the same for both MAP and REDUCE phases. - String sql = "SELECT SUM(s.id), SUM(h.id) FROM SMALL s JOIN HUGE h on h.id = s.id"; - - IgniteRel phys = physicalPlan(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin"); - - assertNotNull(phys); - - IgniteNestedLoopJoin join = findFirstNode(phys, byClass(IgniteNestedLoopJoin.class)); - assertNotNull(join); - - IgniteProject proj = findFirstNode(phys, byClass(IgniteProject.class)); - assertNotNull(proj); - - IgniteTableScan rightScan = findFirstNode(join.getRight(), byClass(IgniteTableScan.class)); - assertNotNull(rightScan); - - IgniteTableScan leftScan = findFirstNode(join.getLeft(), byClass(IgniteTableScan.class)); - assertNotNull(leftScan); - - List rightSchemaWithName = rightScan.getTable().getQualifiedName(); - - assertEquals(2, rightSchemaWithName.size()); - - assertEquals(rightSchemaWithName.get(1), "SMALL"); - - List leftSchemaWithName = leftScan.getTable().getQualifiedName(); - - assertEquals(leftSchemaWithName.get(1), "HUGE"); - - assertEquals(JoinRelType.INNER, join.getJoinType()); - - PlanningContext ctx = plannerCtx(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin"); - - RelOptPlanner pl = ctx.cluster().getPlanner(); - - RelOptCost costWithCommute = pl.getCost(phys, phys.getCluster().getMetadataQuery()); - assertNotNull(costWithCommute); - - System.out.println("plan: " + RelOptUtil.toString(phys)); - - assertNotNull(phys); - - phys = physicalPlan(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin", "JoinCommuteRule"); - - join = findFirstNode(phys, byClass(IgniteNestedLoopJoin.class)); - proj = findFirstNode(phys, byClass(IgniteProject.class)); - - rightScan = findFirstNode(join.getRight(), byClass(IgniteTableScan.class)); - leftScan = findFirstNode(join.getLeft(), byClass(IgniteTableScan.class)); - - assertNotNull(join); - assertNull(proj); - assertNotNull(rightScan); - assertNotNull(leftScan); - - rightSchemaWithName = rightScan.getTable().getQualifiedName(); - - assertEquals(2, rightSchemaWithName.size()); - // no commute - assertEquals(rightSchemaWithName.get(1), "HUGE"); - - leftSchemaWithName = leftScan.getTable().getQualifiedName(); - - assertEquals(leftSchemaWithName.get(1), "SMALL"); - - // no commute - assertEquals(JoinRelType.INNER, join.getJoinType()); - - ctx = plannerCtx(sql, publicSchema, "HashJoinConverter", "MergeJoinConverter", "CorrelatedNestedLoopJoin", "JoinCommuteRule"); - - pl = ctx.cluster().getPlanner(); - - RelOptCost costWithoutCommute = pl.getCost(phys, phys.getCluster().getMetadataQuery()); - - System.out.println("plan: " + RelOptUtil.toString(phys)); - - assertTrue(costWithCommute.isLt(costWithoutCommute)); - } - - /** - * The test verifies that queries with a considerable number of joins can be planned for a - * reasonable amount of time, thus all assertions ensure that the planer returns anything but - * null. The "reasonable amount of time" here is a timeout of the test. With enabled - * {@link CoreRules#JOIN_COMMUTE}, optimization of joining with a few dozens of tables will - * take an eternity. Thus, if the test finished before it was killed, it can be considered - * a success. - */ - @Test - public void commuteIsDisabledForBigJoinsOfTables() throws Exception { - int joinSize = 50; - - IgniteTable[] tables = new IgniteTable[joinSize]; - - for (int i = 0; i < joinSize; i++) { - tables[i] = TestBuilders.table() - .name("T" + (i + 1)) - .addColumn("ID", NativeTypes.INT32) - .addColumn("AFF", NativeTypes.INT32) - .addColumn("VAL", NativeTypes.stringOf(128)) - .distribution(IgniteDistributions.hash(List.of(1))) - .build(); - } - - IgniteSchema schema = createSchema(tables); - - String tableList = IntStream.range(1, joinSize + 1) - .mapToObj(i -> "t" + i) // t1, t2, t3... - .collect(Collectors.joining(", ")); - - String predicateList = IntStream.range(1, joinSize) - .mapToObj(i -> format("t{}.id = t{}.{}", i, i + 1, i % 3 == 0 ? "aff" : "id")) - .collect(Collectors.joining(" AND ")); - - String queryWithBigJoin = format("SELECT t1.val FROM {} WHERE {}", tableList, predicateList); - - { - IgniteRel root = physicalPlan(queryWithBigJoin, schema); - - assertNotNull(root); - } - - { - IgniteRel root = physicalPlan(format("SELECT ({}) as c", queryWithBigJoin), schema); - - assertNotNull(root); - } - - { - IgniteRel root = physicalPlan(format("SELECT 1 FROM t1 WHERE ({}) like 'foo%'", queryWithBigJoin), schema); - - assertNotNull(root); - } - } - - /** - * The same as {@link #commuteIsDisabledForBigJoinsOfTables()}, but with table functions as source of data. - */ - @Test - public void commuteIsDisabledForBigJoinsOfTableFunctions() throws Exception { - int joinSize = 50; - - IgniteSchema schema = createSchema(); - - String tableList = IntStream.range(1, joinSize + 1) - .mapToObj(i -> format("table(system_range(0, 10)) t{}(x)", i)) - .collect(Collectors.joining(", ")); - - String predicateList = IntStream.range(1, joinSize) - .mapToObj(i -> format("t{}.x = t{}.x", i, i + 1)) - .collect(Collectors.joining(" AND ")); - - String queryWithBigJoin = format("SELECT t1.x FROM {} WHERE {}", tableList, predicateList); - - { - IgniteRel root = physicalPlan(queryWithBigJoin, schema); - - assertNotNull(root); - } - - { - IgniteRel root = physicalPlan(format("SELECT ({}) as c", queryWithBigJoin), schema); - - assertNotNull(root); - } - - { - IgniteRel root = physicalPlan(format("SELECT 1 FROM table(system_range(0, 1)) WHERE ({}) > 0", queryWithBigJoin), schema); - - assertNotNull(root); - } - } -} diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpcdsQueryPlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpcdsQueryPlannerTest.java new file mode 100644 index 00000000000..dcd4165f9ff --- /dev/null +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpcdsQueryPlannerTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.planner; + +import static org.apache.ignite.internal.sql.engine.planner.AbstractTpcQueryPlannerTest.TpcSuiteInfo; + +import org.apache.ignite.internal.sql.engine.util.tpcds.TpcdsHelper; +import org.apache.ignite.internal.sql.engine.util.tpcds.TpcdsTables; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Tests ensures a planner generates optimal plan for TPC-DS queries. + */ +// TODO https://issues.apache.org/jira/browse/IGNITE-21986 validate other query plans and make test parameterized. +@TpcSuiteInfo( + tables = TpcdsTables.class, + queryLoader = "getQueryString", + planLoader = "getQueryPlan" +) +public class TpcdsQueryPlannerTest extends AbstractTpcQueryPlannerTest { + @ParameterizedTest + @ValueSource(strings = "64") + public void test(String queryId) { + validateQueryPlan(queryId); + } + + @SuppressWarnings("unused") // used reflectively by AbstractTpcQueryPlannerTest + static String getQueryString(String queryId) { + return TpcdsHelper.getQuery(queryId); + } + + @SuppressWarnings("unused") // used reflectively by AbstractTpcQueryPlannerTest + static String getQueryPlan(String queryId) { + // variant query ends with "v" + boolean variant = queryId.endsWith("v"); + int numericId; + + if (variant) { + String idString = queryId.substring(0, queryId.length() - 1); + numericId = Integer.parseInt(idString); + } else { + numericId = Integer.parseInt(queryId); + } + + if (variant) { + var variantQueryFile = String.format("tpcds/plan/variant_q%d.plan", numericId); + return loadFromResource(variantQueryFile); + } else { + var queryFile = String.format("tpcds/plan/q%s.plan", numericId); + return loadFromResource(queryFile); + } + } +} diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpchQueryPlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpchQueryPlannerTest.java index 6fb4329ea9c..1d59657115e 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpchQueryPlannerTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/TpchQueryPlannerTest.java @@ -17,27 +17,12 @@ package org.apache.ignite.internal.sql.engine.planner; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.apache.ignite.internal.sql.engine.planner.AbstractTpcQueryPlannerTest.TpcSuiteInfo; -import com.google.common.io.CharStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.sql.SqlExplainLevel; -import org.apache.ignite.internal.sql.engine.framework.TestBuilders; -import org.apache.ignite.internal.sql.engine.framework.TestCluster; -import org.apache.ignite.internal.sql.engine.framework.TestNode; -import org.apache.ignite.internal.sql.engine.prepare.MultiStepPlan; -import org.apache.ignite.internal.sql.engine.util.Cloner; -import org.apache.ignite.internal.sql.engine.util.Commons; import org.apache.ignite.internal.sql.engine.util.tpch.TpchHelper; import org.apache.ignite.internal.sql.engine.util.tpch.TpchTables; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Tests ensures a planner generates optimal plan for TPC-H queries. @@ -45,47 +30,32 @@ * @see org.apache.ignite.internal.sql.engine.benchmarks.TpchParseBenchmark */ // TODO https://issues.apache.org/jira/browse/IGNITE-21986 validate other query plans and make test parameterized. -public class TpchQueryPlannerTest extends AbstractPlannerTest { - private static TestCluster CLUSTER; - - @BeforeAll - static void startCluster() { - CLUSTER = TestBuilders.cluster().nodes("N1").build(); - CLUSTER.start(); - - TestNode node = CLUSTER.node("N1"); - - node.initSchema(TpchTables.LINEITEM.ddlScript()); +@TpcSuiteInfo( + tables = TpchTables.class, + queryLoader = "getQueryString", + planLoader = "getQueryPlan" +) +public class TpchQueryPlannerTest extends AbstractTpcQueryPlannerTest { + @ParameterizedTest + // TODO: https://issues.apache.org/jira/browse/IGNITE-24195 + // Query #7 contains disjunctive predicate that switches its part from run to run, making + // the test unstable. + // + // HashJoin(condition=[... OR(=($10, _UTF-8'GERMANY'), =($14, _UTF-8'GERMANY')), OR(=($14, _UTF-8'FRANCE'), =($10, _UTF-8'FRANCE')))] + // vs + // HashJoin(condition=[... OR(=($14, _UTF-8'FRANCE'), =($10, _UTF-8'FRANCE')), OR(=($10, _UTF-8'GERMANY'), =($14, _UTF-8'GERMANY')))] + @ValueSource(strings = {"1", "5", /* "7", */ "8", "9"}) + public void test(String queryId) { + validateQueryPlan(queryId); } - @AfterAll - static void stopCluster() throws Exception { - CLUSTER.stop(); - CLUSTER = null; + @SuppressWarnings("unused") // used reflectively by AbstractTpcQueryPlannerTest + static String getQueryString(String queryId) { + return TpchHelper.getQuery(queryId); } - @Test - public void tpchTest_q1() { - validateQueryPlan("1"); - } - - private static void validateQueryPlan(String queryId) { - TestNode node = CLUSTER.node("N1"); - - MultiStepPlan plan = (MultiStepPlan) node.prepare(TpchHelper.getQuery(queryId)); - - String actualPlan = RelOptUtil.toString(Cloner.clone(plan.root(), Commons.cluster()), SqlExplainLevel.DIGEST_ATTRIBUTES); - String expectedPlan = getQueryPlan(queryId); - - assertEquals(expectedPlan, actualPlan); - } - - /** - * Loads query plan for provided TPC-H query id. - * - * @see TpchHelper#getQuery(String) for query id details. - */ - public static String getQueryPlan(String queryId) { + @SuppressWarnings("unused") // used reflectively by AbstractTpcQueryPlannerTest + static String getQueryPlan(String queryId) { // variant query ends with "v" boolean variant = queryId.endsWith("v"); int numericId; @@ -105,17 +75,4 @@ public static String getQueryPlan(String queryId) { return loadFromResource(queryFile); } } - - static String loadFromResource(String resource) { - try (InputStream is = TpchHelper.class.getClassLoader().getResourceAsStream(resource)) { - if (is == null) { - throw new IllegalArgumentException("Resource does not exist: " + resource); - } - try (InputStreamReader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) { - return CharStreams.toString(reader); - } - } catch (IOException e) { - throw new UncheckedIOException("I/O operation failed: " + resource, e); - } - } } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImplTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImplTest.java index 636b882ec4e..410ba4b1339 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImplTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/statistic/SqlStatisticManagerImplTest.java @@ -85,22 +85,6 @@ public void checkDefaultTableSize() { assertEquals(DEFAULT_TABLE_SIZE, sqlStatisticManager.tableSize(tableId)); } - @Test - public void checkMinimumTableSize() { - int tableId = ThreadLocalRandom.current().nextInt(); - prepareCatalogWithTable(tableId); - - when(tableManager.cachedTable(tableId)).thenReturn(tableViewInternal); - when(tableViewInternal.internalTable()).thenReturn(internalTable); - when(internalTable.estimatedSize()).thenReturn(CompletableFuture.completedFuture(10L)); - - SqlStatisticManagerImpl sqlStatisticManager = new SqlStatisticManagerImpl(tableManager, catalogManager, lowWatermark); - sqlStatisticManager.start(); - - // even table size is 10 it should return default minimum size - assertEquals(1_000L, sqlStatisticManager.tableSize(tableId)); - } - @Test public void checkTableSize() { int tableId = ThreadLocalRandom.current().nextInt(); diff --git a/modules/sql-engine/src/test/resources/mapping/table_identity.test b/modules/sql-engine/src/test/resources/mapping/table_identity.test index e13befad791..cf4a9ad6a4a 100644 --- a/modules/sql-engine/src/test/resources/mapping/table_identity.test +++ b/modules/sql-engine/src/test/resources/mapping/table_identity.test @@ -4,29 +4,30 @@ SELECT * FROM nt1_n1, nt2_n2 Fragment#0 root executionNodes: [N0] remoteFragments: [1, 2] - exchangeSourceNodes: {1=[N1], 2=[N2]} + exchangeSourceNodes: {1=[N2], 2=[N1]} tree: - NestedLoopJoin - Receiver(sourceFragment=1, exchange=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + Receiver(sourceFragment=2, exchange=2, distribution=single) Fragment#1 targetNodes: [N0] - executionNodes: [N1] - tables: [NT1_N1] - partitions: {N1=[0:1]} + executionNodes: [N2] + tables: [NT2_N2] + partitions: {N2=[0:1]} tree: Sender(targetFragment=0, exchange=1, distribution=single) - TableScan(name=PUBLIC.NT1_N1, source=4, partitions=1, distribution=identity[0]) + TableScan(name=PUBLIC.NT2_N2, source=4, partitions=1, distribution=identity[0]) Fragment#2 targetNodes: [N0] - executionNodes: [N2] - tables: [NT2_N2] - partitions: {N2=[0:1]} + executionNodes: [N1] + tables: [NT1_N1] + partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=2, distribution=single) - TableScan(name=PUBLIC.NT2_N2, source=3, partitions=1, distribution=identity[0]) + TableScan(name=PUBLIC.NT1_N1, source=3, partitions=1, distribution=identity[0]) --- N1 @@ -35,29 +36,30 @@ SELECT * FROM nt1_n1, nt2_n2 Fragment#0 root executionNodes: [N1] remoteFragments: [1, 2] - exchangeSourceNodes: {1=[N1], 2=[N2]} + exchangeSourceNodes: {1=[N2], 2=[N1]} tree: - NestedLoopJoin - Receiver(sourceFragment=1, exchange=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + Receiver(sourceFragment=2, exchange=2, distribution=single) Fragment#1 targetNodes: [N1] - executionNodes: [N1] - tables: [NT1_N1] - partitions: {N1=[0:1]} + executionNodes: [N2] + tables: [NT2_N2] + partitions: {N2=[0:1]} tree: Sender(targetFragment=0, exchange=1, distribution=single) - TableScan(name=PUBLIC.NT1_N1, source=4, partitions=1, distribution=identity[0]) + TableScan(name=PUBLIC.NT2_N2, source=4, partitions=1, distribution=identity[0]) Fragment#2 targetNodes: [N1] - executionNodes: [N2] - tables: [NT2_N2] - partitions: {N2=[0:1]} + executionNodes: [N1] + tables: [NT1_N1] + partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=2, distribution=single) - TableScan(name=PUBLIC.NT2_N2, source=3, partitions=1, distribution=identity[0]) + TableScan(name=PUBLIC.NT1_N1, source=3, partitions=1, distribution=identity[0]) --- N0 @@ -68,27 +70,28 @@ Fragment#0 root remoteFragments: [1, 2] exchangeSourceNodes: {1=[N1], 2=[N1]} tree: - NestedLoopJoin - Receiver(sourceFragment=1, exchange=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + Receiver(sourceFragment=2, exchange=2, distribution=single) Fragment#1 targetNodes: [N0] executionNodes: [N1] - tables: [NT1_N1] + tables: [NT2_N1] partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=1, distribution=single) - TableScan(name=PUBLIC.NT1_N1, source=4, partitions=1, distribution=identity[0]) + TableScan(name=PUBLIC.NT2_N1, source=4, partitions=1, distribution=identity[0]) Fragment#2 targetNodes: [N0] executionNodes: [N1] - tables: [NT2_N1] + tables: [NT1_N1] partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=2, distribution=single) - TableScan(name=PUBLIC.NT2_N1, source=3, partitions=1, distribution=identity[0]) + TableScan(name=PUBLIC.NT1_N1, source=3, partitions=1, distribution=identity[0]) --- N1 @@ -99,25 +102,26 @@ Fragment#0 root remoteFragments: [1, 2] exchangeSourceNodes: {1=[N1], 2=[N1]} tree: - NestedLoopJoin - Receiver(sourceFragment=1, exchange=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + Receiver(sourceFragment=2, exchange=2, distribution=single) Fragment#1 targetNodes: [N1] executionNodes: [N1] - tables: [NT1_N1] + tables: [NT2_N1] partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=1, distribution=single) - TableScan(name=PUBLIC.NT1_N1, source=4, partitions=1, distribution=identity[0]) + TableScan(name=PUBLIC.NT2_N1, source=4, partitions=1, distribution=identity[0]) Fragment#2 targetNodes: [N1] executionNodes: [N1] - tables: [NT2_N1] + tables: [NT1_N1] partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=2, distribution=single) - TableScan(name=PUBLIC.NT2_N1, source=3, partitions=1, distribution=identity[0]) ---- + TableScan(name=PUBLIC.NT1_N1, source=3, partitions=1, distribution=identity[0]) +--- \ No newline at end of file diff --git a/modules/sql-engine/src/test/resources/mapping/table_identity_single.test b/modules/sql-engine/src/test/resources/mapping/table_identity_single.test index ce1d33459bf..934e24efde4 100644 --- a/modules/sql-engine/src/test/resources/mapping/table_identity_single.test +++ b/modules/sql-engine/src/test/resources/mapping/table_identity_single.test @@ -11,23 +11,24 @@ Fragment#0 root Fragment#4 targetNodes: [N0] executionNodes: [N1] - remoteFragments: [2] - exchangeSourceNodes: {2=[N1]} + remoteFragments: [1] + exchangeSourceNodes: {1=[N1]} tables: [CT_N1] partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=4, distribution=single) - NestedLoopJoin - TableScan(name=PUBLIC.CT_N1, source=1, partitions=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + TableScan(name=PUBLIC.CT_N1, source=2, partitions=1, distribution=single) -Fragment#2 +Fragment#1 targetNodes: [N1] executionNodes: [N1] tables: [NT_N1] partitions: {N1=[0:1]} tree: - Sender(targetFragment=4, exchange=2, distribution=single) + Sender(targetFragment=4, exchange=1, distribution=single) TableScan(name=PUBLIC.NT_N1, source=3, partitions=1, distribution=identity[0]) --- @@ -36,22 +37,23 @@ SELECT * FROM CT_n1, NT_n1 --- Fragment#0 root executionNodes: [N1] - remoteFragments: [2] - exchangeSourceNodes: {2=[N1]} + remoteFragments: [1] + exchangeSourceNodes: {1=[N1]} tables: [CT_N1] partitions: {N1=[0:1]} tree: - NestedLoopJoin - TableScan(name=PUBLIC.CT_N1, source=1, partitions=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + TableScan(name=PUBLIC.CT_N1, source=2, partitions=1, distribution=single) -Fragment#2 +Fragment#1 targetNodes: [N1] executionNodes: [N1] tables: [NT_N1] partitions: {N1=[0:1]} tree: - Sender(targetFragment=0, exchange=2, distribution=single) + Sender(targetFragment=0, exchange=1, distribution=single) TableScan(name=PUBLIC.NT_N1, source=3, partitions=1, distribution=identity[0]) --- @@ -68,23 +70,24 @@ Fragment#0 root Fragment#4 targetNodes: [N0] executionNodes: [N1] - remoteFragments: [2] - exchangeSourceNodes: {2=[N2]} + remoteFragments: [1] + exchangeSourceNodes: {1=[N2]} tables: [CT_N1] partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=4, distribution=single) - NestedLoopJoin - TableScan(name=PUBLIC.CT_N1, source=1, partitions=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + TableScan(name=PUBLIC.CT_N1, source=2, partitions=1, distribution=single) -Fragment#2 +Fragment#1 targetNodes: [N1] executionNodes: [N2] tables: [NT_N2] partitions: {N2=[0:1]} tree: - Sender(targetFragment=4, exchange=2, distribution=single) + Sender(targetFragment=4, exchange=1, distribution=single) TableScan(name=PUBLIC.NT_N2, source=3, partitions=1, distribution=identity[0]) --- @@ -93,21 +96,22 @@ SELECT * FROM CT_n1, NT_n2 --- Fragment#0 root executionNodes: [N1] - remoteFragments: [2] - exchangeSourceNodes: {2=[N2]} + remoteFragments: [1] + exchangeSourceNodes: {1=[N2]} tables: [CT_N1] partitions: {N1=[0:1]} tree: - NestedLoopJoin - TableScan(name=PUBLIC.CT_N1, source=1, partitions=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + NestedLoopJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + TableScan(name=PUBLIC.CT_N1, source=2, partitions=1, distribution=single) -Fragment#2 +Fragment#1 targetNodes: [N1] executionNodes: [N2] tables: [NT_N2] partitions: {N2=[0:1]} tree: - Sender(targetFragment=0, exchange=2, distribution=single) + Sender(targetFragment=0, exchange=1, distribution=single) TableScan(name=PUBLIC.NT_N2, source=3, partitions=1, distribution=identity[0]) --- diff --git a/modules/sql-engine/src/test/resources/mapping/table_single.test b/modules/sql-engine/src/test/resources/mapping/table_single.test index 7a2aa7ea1da..3758ab52763 100644 --- a/modules/sql-engine/src/test/resources/mapping/table_single.test +++ b/modules/sql-engine/src/test/resources/mapping/table_single.test @@ -15,9 +15,10 @@ Fragment#3 partitions: {N1=[0:1]} tree: Sender(targetFragment=0, exchange=3, distribution=single) - NestedLoopJoin - TableScan(name=PUBLIC.CT1_N1, source=1, partitions=1, distribution=single) - TableScan(name=PUBLIC.CT2_N1, source=2, partitions=1, distribution=single) + Project + NestedLoopJoin + TableScan(name=PUBLIC.CT2_N1, source=1, partitions=1, distribution=single) + TableScan(name=PUBLIC.CT1_N1, source=2, partitions=1, distribution=single) --- N1 @@ -28,9 +29,10 @@ Fragment#0 root tables: [CT1_N1, CT2_N1] partitions: {N1=[0:1]} tree: - NestedLoopJoin - TableScan(name=PUBLIC.CT1_N1, source=1, partitions=1, distribution=single) - TableScan(name=PUBLIC.CT2_N1, source=2, partitions=1, distribution=single) + Project + NestedLoopJoin + TableScan(name=PUBLIC.CT2_N1, source=1, partitions=1, distribution=single) + TableScan(name=PUBLIC.CT1_N1, source=2, partitions=1, distribution=single) --- N0 @@ -52,9 +54,10 @@ Fragment#4 partitions: {N2=[0:1]} tree: Sender(targetFragment=0, exchange=4, distribution=single) - NestedLoopJoin - Receiver(sourceFragment=3, exchange=3, distribution=single) - TableScan(name=PUBLIC.CT2_N2, source=2, partitions=1, distribution=single) + Project + NestedLoopJoin + TableScan(name=PUBLIC.CT2_N2, source=1, partitions=1, distribution=single) + Receiver(sourceFragment=3, exchange=3, distribution=single) Fragment#3 targetNodes: [N2] @@ -63,7 +66,7 @@ Fragment#3 partitions: {N1=[0:1]} tree: Sender(targetFragment=4, exchange=3, distribution=single) - TableScan(name=PUBLIC.CT1_N1, source=1, partitions=1, distribution=single) + TableScan(name=PUBLIC.CT1_N1, source=2, partitions=1, distribution=single) --- N1 @@ -85,9 +88,10 @@ Fragment#4 partitions: {N2=[0:1]} tree: Sender(targetFragment=0, exchange=4, distribution=single) - NestedLoopJoin - Receiver(sourceFragment=3, exchange=3, distribution=single) - TableScan(name=PUBLIC.CT2_N2, source=2, partitions=1, distribution=single) + Project + NestedLoopJoin + TableScan(name=PUBLIC.CT2_N2, source=1, partitions=1, distribution=single) + Receiver(sourceFragment=3, exchange=3, distribution=single) Fragment#3 targetNodes: [N2] @@ -96,5 +100,5 @@ Fragment#3 partitions: {N1=[0:1]} tree: Sender(targetFragment=4, exchange=3, distribution=single) - TableScan(name=PUBLIC.CT1_N1, source=1, partitions=1, distribution=single) + TableScan(name=PUBLIC.CT1_N1, source=2, partitions=1, distribution=single) --- diff --git a/modules/sql-engine/src/test/resources/mapping/test_partition_pruning.test b/modules/sql-engine/src/test/resources/mapping/test_partition_pruning.test index 8d326c6f17d..cd5bd9255ac 100644 --- a/modules/sql-engine/src/test/resources/mapping/test_partition_pruning.test +++ b/modules/sql-engine/src/test/resources/mapping/test_partition_pruning.test @@ -25,31 +25,32 @@ SELECT /*+ DISABLE_RULE('NestedLoopJoinConverter', 'HashJoinConverter', 'Correla Fragment#0 root executionNodes: [N1] remoteFragments: [1, 2] - exchangeSourceNodes: {1=[N2], 2=[N4]} + exchangeSourceNodes: {1=[N4], 2=[N2]} tree: - MergeJoin - Receiver(sourceFragment=1, exchange=1, distribution=single) - Receiver(sourceFragment=2, exchange=2, distribution=single) + Project + MergeJoin + Receiver(sourceFragment=1, exchange=1, distribution=single) + Receiver(sourceFragment=2, exchange=2, distribution=single) Fragment#1 targetNodes: [N1] - executionNodes: [N2] - tables: [T1_N1N2N3] - partitions: {N2=[1:3]} + executionNodes: [N4] + tables: [T2_N4N5] + partitions: {N4=[0:2]} tree: Sender(targetFragment=0, exchange=1, distribution=single) Sort - TableScan(name=PUBLIC.T1_N1N2N3, source=4, partitions=3, distribution=affinity[table: T1_N1N2N3, columns: [ID]]) + TableScan(name=PUBLIC.T2_N4N5, source=4, partitions=2, distribution=affinity[table: T2_N4N5, columns: [ID]]) Fragment#2 targetNodes: [N1] - executionNodes: [N4] - tables: [T2_N4N5] - partitions: {N4=[0:2]} + executionNodes: [N2] + tables: [T1_N1N2N3] + partitions: {N2=[1:3]} tree: Sender(targetFragment=0, exchange=2, distribution=single) Sort - TableScan(name=PUBLIC.T2_N4N5, source=3, partitions=2, distribution=affinity[table: T2_N4N5, columns: [ID]]) + TableScan(name=PUBLIC.T1_N1N2N3, source=3, partitions=3, distribution=affinity[table: T1_N1N2N3, columns: [ID]]) --- # Self join, different predicates that produce same set of partitions N1 diff --git a/modules/sql-engine/src/test/resources/tpcds/plan/q64.plan b/modules/sql-engine/src/test/resources/tpcds/plan/q64.plan new file mode 100644 index 00000000000..589d3f43f4f --- /dev/null +++ b/modules/sql-engine/src/test/resources/tpcds/plan/q64.plan @@ -0,0 +1,127 @@ +Sort(sort0=[$0], sort1=[$1], sort2=[$20], sort3=[$13], sort4=[$16], dir0=[ASC], dir1=[ASC], dir2=[ASC], dir3=[ASC], dir4=[ASC]) + Project(inputs=[0], exprs=[[$2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $15, $16, $17, $18, $35, $36, $37, $31, $34]]) + HashJoin(condition=[AND(=($1, $20), <=($34, $15), =($2, $21), =($3, $22))], joinType=[inner]) + ColocatedHashAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}], CNT=[COUNT()], S1=[SUM($15)], S2=[SUM($16)], S3=[SUM($17)]) + Project(exprs=[[$45, $42, $55, $56, $7, $8, $9, $10, $12, $13, $14, $15, $47, $3, $5, $39, $40, $41]]) + HashJoin(condition=[AND(=($31, $0), =($38, $1))], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, STORE_RETURNS]], requiredColumns=[{2, 9}]) + HashJoin(condition=[=($27, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, DATE_DIM]], requiredColumns=[{0, 6}]) + HashJoin(condition=[=($24, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, DATE_DIM]], requiredColumns=[{0, 6}]) + HashJoin(condition=[=($30, $48)], joinType=[inner]) + HashJoin(condition=[=($31, $47)], joinType=[inner]) + HashJoin(condition=[=($29, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_ADDRESS]], requiredColumns=[{0, 2, 3, 6, 9}]) + HashJoin(condition=[=($16, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_ADDRESS]], requiredColumns=[{0, 2, 3, 6, 9}]) + HashJoin(condition=[=($1, $36)], joinType=[inner]) + HashJoin(condition=[=($18, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, HOUSEHOLD_DEMOGRAPHICS]], requiredColumns=[{0, 1}]) + HashJoin(condition=[=($1, $33)], joinType=[inner]) + HashJoin(condition=[=($8, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, HOUSEHOLD_DEMOGRAPHICS]], requiredColumns=[{0, 1}]) + HashJoin(condition=[AND(<>($1, $3), =($13, $0))], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_DEMOGRAPHICS]], requiredColumns=[{0, 2}]) + HashJoin(condition=[=($3, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_DEMOGRAPHICS]], requiredColumns=[{0, 2}]) + HashJoin(condition=[=($8, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER]], requiredColumns=[{0, 2, 3, 4, 5, 6}]) + HashJoin(condition=[=($1, $18)], joinType=[inner]) + HashJoin(condition=[=($0, $16)], joinType=[inner]) + HashJoin(condition=[=($1, $12)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, STORE_SALES]], requiredColumns=[{0, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 19}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, ITEM]], filters=[AND(OR(=($t2, _UTF-8'azure'), =($t2, _UTF-8'blush'), =($t2, _UTF-8'gainsboro'), =($t2, _UTF-8'hot'), =($t2, _UTF-8'lemon'), =($t2, _UTF-8'misty')), >=(CAST($t1):DECIMAL(10, 2), 80.00), <=(CAST($t1):DECIMAL(10, 2), CAST(+(80, 10)):DECIMAL(10, 2) NOT NULL), >=(CAST($t1):DECIMAL(10, 2), CAST(+(80, 1)):DECIMAL(10, 2) NOT NULL), <=(CAST($t1):DECIMAL(10, 2), CAST(+(80, 15)):DECIMAL(10, 2) NOT NULL))], requiredColumns=[{0, 5, 17, 21}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, DATE_DIM]], filters=[=($t1, 1999)], requiredColumns=[{0, 6}]) + Filter(condition=[>($1, *(2, $2))]) + ReduceHashAggregate(group=[{0}], SALE=[SUM($1)], REFUND=[SUM($2)]) + Exchange(distribution=[single]) + MapHashAggregate(group=[{0}], SALE=[SUM($1)], REFUND=[SUM($2)]) + Project(inputs=[0], exprs=[[$2, +(+($5, $6), $7)]]) + HashJoin(condition=[AND(=($0, $3), =($1, $4))], joinType=[inner]) + TableScan(table=[[PUBLIC, CATALOG_SALES]], requiredColumns=[{15, 17, 25}]) + Exchange(distribution=[affinity[tableId=51, zoneId=51][0, 1]]) + TableScan(table=[[PUBLIC, CATALOG_RETURNS]], requiredColumns=[{2, 16, 23, 24, 25}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, INCOME_BAND]], requiredColumns=[{0}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, INCOME_BAND]], requiredColumns=[{0}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, PROMOTION]], requiredColumns=[{0}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, STORE]], requiredColumns=[{0, 5, 25}]) + ColocatedHashAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}], CNT=[COUNT()], S1=[SUM($15)], S2=[SUM($16)], S3=[SUM($17)]) + Project(exprs=[[$45, $42, $55, $56, $7, $8, $9, $10, $12, $13, $14, $15, $47, $3, $5, $39, $40, $41]]) + HashJoin(condition=[AND(=($31, $0), =($38, $1))], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, STORE_RETURNS]], requiredColumns=[{2, 9}]) + HashJoin(condition=[=($27, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, DATE_DIM]], requiredColumns=[{0, 6}]) + HashJoin(condition=[=($24, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, DATE_DIM]], requiredColumns=[{0, 6}]) + HashJoin(condition=[=($30, $48)], joinType=[inner]) + HashJoin(condition=[=($31, $47)], joinType=[inner]) + HashJoin(condition=[=($29, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_ADDRESS]], requiredColumns=[{0, 2, 3, 6, 9}]) + HashJoin(condition=[=($16, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_ADDRESS]], requiredColumns=[{0, 2, 3, 6, 9}]) + HashJoin(condition=[=($1, $36)], joinType=[inner]) + HashJoin(condition=[=($18, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, HOUSEHOLD_DEMOGRAPHICS]], requiredColumns=[{0, 1}]) + HashJoin(condition=[=($1, $33)], joinType=[inner]) + HashJoin(condition=[=($8, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, HOUSEHOLD_DEMOGRAPHICS]], requiredColumns=[{0, 1}]) + HashJoin(condition=[AND(<>($1, $3), =($13, $0))], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_DEMOGRAPHICS]], requiredColumns=[{0, 2}]) + HashJoin(condition=[=($3, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER_DEMOGRAPHICS]], requiredColumns=[{0, 2}]) + HashJoin(condition=[=($8, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER]], requiredColumns=[{0, 2, 3, 4, 5, 6}]) + HashJoin(condition=[=($1, $18)], joinType=[inner]) + HashJoin(condition=[=($0, $16)], joinType=[inner]) + HashJoin(condition=[=($1, $12)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, STORE_SALES]], requiredColumns=[{0, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 19}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, ITEM]], filters=[AND(OR(=($t2, _UTF-8'azure'), =($t2, _UTF-8'blush'), =($t2, _UTF-8'gainsboro'), =($t2, _UTF-8'hot'), =($t2, _UTF-8'lemon'), =($t2, _UTF-8'misty')), >=(CAST($t1):DECIMAL(10, 2), 80.00), <=(CAST($t1):DECIMAL(10, 2), CAST(+(80, 10)):DECIMAL(10, 2) NOT NULL), >=(CAST($t1):DECIMAL(10, 2), CAST(+(80, 1)):DECIMAL(10, 2) NOT NULL), <=(CAST($t1):DECIMAL(10, 2), CAST(+(80, 15)):DECIMAL(10, 2) NOT NULL))], requiredColumns=[{0, 5, 17, 21}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, DATE_DIM]], filters=[=($t1, +(1999, 1))], requiredColumns=[{0, 6}]) + Filter(condition=[>($1, *(2, $2))]) + ReduceHashAggregate(group=[{0}], SALE=[SUM($1)], REFUND=[SUM($2)]) + Exchange(distribution=[single]) + MapHashAggregate(group=[{0}], SALE=[SUM($1)], REFUND=[SUM($2)]) + Project(inputs=[0], exprs=[[$2, +(+($5, $6), $7)]]) + HashJoin(condition=[AND(=($0, $3), =($1, $4))], joinType=[inner]) + TableScan(table=[[PUBLIC, CATALOG_SALES]], requiredColumns=[{15, 17, 25}]) + Exchange(distribution=[affinity[tableId=51, zoneId=51][0, 1]]) + TableScan(table=[[PUBLIC, CATALOG_RETURNS]], requiredColumns=[{2, 16, 23, 24, 25}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, INCOME_BAND]], requiredColumns=[{0}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, INCOME_BAND]], requiredColumns=[{0}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, PROMOTION]], requiredColumns=[{0}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, STORE]], requiredColumns=[{0, 5, 25}]) diff --git a/modules/sql-engine/src/test/resources/tpch/plan/q5.plan b/modules/sql-engine/src/test/resources/tpch/plan/q5.plan new file mode 100644 index 00000000000..b4967917d9a --- /dev/null +++ b/modules/sql-engine/src/test/resources/tpch/plan/q5.plan @@ -0,0 +1,20 @@ +Sort(sort0=[$1], dir0=[DESC]) + ColocatedHashAggregate(group=[{0}], REVENUE=[SUM($1)]) + Project(exprs=[[$7, *($2, -(1, $3))]]) + HashJoin(condition=[AND(=($15, $5), =($14, $12))], joinType=[inner]) + HashJoin(condition=[=($0, $11)], joinType=[inner]) + HashJoin(condition=[=($1, $4)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, LINEITEM]], requiredColumns=[{0, 2, 5, 6}]) + HashJoin(condition=[=($1, $2)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, SUPPLIER]], requiredColumns=[{0, 3}]) + NestedLoopJoin(condition=[=($2, $3)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, NATION]], requiredColumns=[{0, 1, 2}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, REGION]], filters=[=($t1, _UTF-8'ASIA')], requiredColumns=[{0, 1}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, ORDERS]], filters=[AND(>=($t2, 1994-01-01), <($t2, +(1994-01-01, 12:INTERVAL YEAR)))], requiredColumns=[{0, 1, 4}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER]], requiredColumns=[{0, 3}]) diff --git a/modules/sql-engine/src/test/resources/tpch/plan/q7.plan b/modules/sql-engine/src/test/resources/tpch/plan/q7.plan new file mode 100644 index 00000000000..51ee7a38591 --- /dev/null +++ b/modules/sql-engine/src/test/resources/tpch/plan/q7.plan @@ -0,0 +1,20 @@ +Sort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]) + ColocatedHashAggregate(group=[{0, 1, 2}], REVENUE=[SUM($3)]) + Project(exprs=[[$14, $10, EXTRACT(FLAG(YEAR), $4), *($2, -(1, $3))]]) + HashJoin(condition=[AND(=($11, $1), OR(=($10, _UTF-8'GERMANY'), =($14, _UTF-8'GERMANY')), OR(=($14, _UTF-8'FRANCE'), =($10, _UTF-8'FRANCE')))], joinType=[inner]) + HashJoin(condition=[=($5, $0)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, LINEITEM]], filters=[AND(>=($t4, 1995-01-01), <=($t4, 1996-12-31))], requiredColumns=[{0, 2, 5, 6, 10}]) + HashJoin(condition=[=($2, $1)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, ORDERS]], requiredColumns=[{0, 1}]) + HashJoin(condition=[=($1, $2)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER]], requiredColumns=[{0, 3}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, NATION]], filters=[OR(=($t1, _UTF-8'FRANCE'), =($t1, _UTF-8'GERMANY'))], requiredColumns=[{0, 1}]) + HashJoin(condition=[=($1, $2)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, SUPPLIER]], requiredColumns=[{0, 3}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, NATION]], filters=[OR(=($t1, _UTF-8'FRANCE'), =($t1, _UTF-8'GERMANY'))], requiredColumns=[{0, 1}]) diff --git a/modules/sql-engine/src/test/resources/tpch/plan/q8.plan b/modules/sql-engine/src/test/resources/tpch/plan/q8.plan new file mode 100644 index 00000000000..46ec98c37f1 --- /dev/null +++ b/modules/sql-engine/src/test/resources/tpch/plan/q8.plan @@ -0,0 +1,27 @@ +Sort(sort0=[$0], dir0=[ASC]) + Project(inputs=[0], exprs=[[/($1, $2)]]) + ColocatedHashAggregate(group=[{0}], agg#0=[SUM($1)], agg#1=[SUM($2)]) + Project(exprs=[[EXTRACT(FLAG(YEAR), $7), CASE(=($19, _UTF-8'BRAZIL'), *($3, -(1, $4)), 0.0000:DECIMAL(31, 4)), *($3, -(1, $4))]]) + HashJoin(condition=[=($16, $2)], joinType=[inner]) + HashJoin(condition=[=($14, $1)], joinType=[inner]) + HashJoin(condition=[=($0, $5)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, LINEITEM]], requiredColumns=[{0, 1, 2, 5, 6}]) + HashJoin(condition=[=($1, $3)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, ORDERS]], filters=[AND(>=($t2, 1995-01-01), <=($t2, 1996-12-31))], requiredColumns=[{0, 1, 4}]) + HashJoin(condition=[=($1, $2)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, CUSTOMER]], requiredColumns=[{0, 3}]) + NestedLoopJoin(condition=[=($1, $2)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, NATION]], requiredColumns=[{0, 2}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, REGION]], filters=[=($t1, _UTF-8'AMERICA')], requiredColumns=[{0, 1}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, PART]], filters=[=($t1, _UTF-8'ECONOMY ANODIZED STEEL')], requiredColumns=[{0, 4}]) + HashJoin(condition=[=($1, $2)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, SUPPLIER]], requiredColumns=[{0, 3}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, NATION]], requiredColumns=[{0, 1}]) diff --git a/modules/sql-engine/src/test/resources/tpch/plan/q9.plan b/modules/sql-engine/src/test/resources/tpch/plan/q9.plan new file mode 100644 index 00000000000..0c22f44544b --- /dev/null +++ b/modules/sql-engine/src/test/resources/tpch/plan/q9.plan @@ -0,0 +1,20 @@ +Sort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[DESC]) + ColocatedHashAggregate(group=[{0, 1}], SUM_PROFIT=[SUM($2)]) + Project(exprs=[[$11, EXTRACT(FLAG(YEAR), $13), -(*($4, -(1, $5)), *($16, $3))]]) + HashJoin(condition=[AND(=($15, $2), =($14, $1))], joinType=[inner]) + HashJoin(condition=[=($12, $0)], joinType=[inner]) + HashJoin(condition=[=($8, $2)], joinType=[inner]) + HashJoin(condition=[=($6, $1)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, LINEITEM]], requiredColumns=[{0, 1, 2, 4, 5, 6}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, PART]], filters=[LIKE($t1, _UTF-8'%green%')], requiredColumns=[{0, 1}]) + HashJoin(condition=[=($1, $2)], joinType=[inner]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, SUPPLIER]], requiredColumns=[{0, 3}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, NATION]], requiredColumns=[{0, 1}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, ORDERS]], requiredColumns=[{0, 4}]) + Exchange(distribution=[single]) + TableScan(table=[[PUBLIC, PARTSUPP]], requiredColumns=[{0, 1, 3}]) diff --git a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/BaseSqlIntegrationTest.java b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/BaseSqlIntegrationTest.java index 4bcce4d66bc..36ce4641916 100644 --- a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/BaseSqlIntegrationTest.java +++ b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/BaseSqlIntegrationTest.java @@ -26,11 +26,15 @@ import static org.junit.jupiter.api.Assertions.assertSame; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.ignite.Ignite; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.sql.engine.AsyncSqlCursor; import org.apache.ignite.internal.sql.engine.SqlQueryProcessor; +import org.apache.ignite.internal.sql.engine.statistic.SqlStatisticManagerImpl; import org.apache.ignite.internal.sql.engine.util.InjectQueryCheckerFactory; import org.apache.ignite.internal.sql.engine.util.QueryChecker; import org.apache.ignite.internal.sql.engine.util.QueryCheckerExtension; @@ -271,6 +275,18 @@ protected void waitUntilRunningQueriesCount(Matcher matcher) { SqlTestUtils.waitUntilRunningQueriesCount(queryProcessor(), matcher); } + protected static void gatherStatistics() { + SqlStatisticManagerImpl statisticManager = (SqlStatisticManagerImpl) ((SqlQueryProcessor) unwrapIgniteImpl(CLUSTER.aliveNode()) + .queryEngine()).sqlStatisticManager(); + + statisticManager.forceUpdateAll(); + try { + statisticManager.lastUpdateStatisticFuture().get(5_000, TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + } + /** An executable that retrieves the data from the specified cursor. */ public static class DrainCursor implements Executable { private final AsyncSqlCursor cursor; diff --git a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcScaleFactor.java b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcScaleFactor.java new file mode 100644 index 00000000000..0e44ae62045 --- /dev/null +++ b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcScaleFactor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; + +/** Enumerates scale factors supported for TPC tables. */ +public enum TpcScaleFactor { + SF_1GB, SF_1TB, SF_3TB, SF_10TB, SF_30TB, SF_100TB; + + /** Returns size that corresponds for current scale factor. */ + public long size(long... sizes) { + assert sizes.length == values().length; + + return sizes[ordinal()]; + } +} diff --git a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcTable.java b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcTable.java index c3b43336e54..e1b39ea9150 100644 --- a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcTable.java +++ b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/TpcTable.java @@ -59,4 +59,7 @@ public interface TpcTable { * @throws IOException In case of error. */ Iterator dataProvider(Path pathToDataset) throws IOException; + + /** Returns estimated size of a table for given scale factor. */ + long estimatedSize(TpcScaleFactor sf); } diff --git a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpcds/TpcdsTables.java b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpcds/TpcdsTables.java index 872ffd079af..58851c57c89 100644 --- a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpcds/TpcdsTables.java +++ b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpcds/TpcdsTables.java @@ -28,6 +28,7 @@ import java.nio.file.Path; import java.time.LocalDate; import java.util.Iterator; +import org.apache.ignite.internal.sql.engine.util.TpcScaleFactor; import org.apache.ignite.internal.sql.engine.util.TpcTable; import org.apache.ignite.internal.sql.engine.util.tpch.TpchHelper; import org.apache.ignite.sql.ColumnType; @@ -38,6 +39,7 @@ @SuppressWarnings("NonSerializableFieldInSerializableClass") public enum TpcdsTables implements TpcTable { CUSTOMER( + new long[] {100_000, 12_000_000, 30_000_000, 65_000_000, 80_000_000, 100_000_000}, new Column("C_CUSTOMER_SK", INT32), new Column("C_CUSTOMER_ID", STRING), new Column("C_CURRENT_CDEMO_SK", INT32), @@ -66,6 +68,7 @@ public String insertPrepareStatement() { }, WEB_PAGE( + new long[] {60, 3_000, 3_600, 4_002, 4_602, 5_004}, new Column("WP_WEB_PAGE_SK", INT32), new Column("WP_WEB_PAGE_ID", STRING), new Column("WP_REC_START_DATE", DATE), @@ -90,6 +93,7 @@ public String insertPrepareStatement() { }, STORE_SALES( + new long[] {2_880_404, 2_879_987_999L, 8_639_936_081L, 28_799_983_563L, 86_399_341_874L, 287_997_818_084L}, new Column("SS_SOLD_DATE_SK", INT32), new Column("SS_SOLD_TIME_SK", INT32), new Column("SS_ITEM_SK", INT32), @@ -125,6 +129,7 @@ public String insertPrepareStatement() { }, HOUSEHOLD_DEMOGRAPHICS( + new long[] {7_200, 7_200, 7_200, 7_200, 7_200, 7_200}, new Column("HD_DEMO_SK", INT32), new Column("HD_INCOME_BAND_SK", INT32), new Column("HD_BUY_POTENTIAL", STRING), @@ -139,6 +144,7 @@ public String insertPrepareStatement() { }, CATALOG_PAGE( + new long[] {11_718, 30_000, 36_000, 40_000, 46_000, 50_000}, new Column("CP_CATALOG_PAGE_SK", INT32), new Column("CP_CATALOG_PAGE_ID", STRING), new Column("CP_START_DATE_SK", INT32), @@ -157,6 +163,7 @@ public String insertPrepareStatement() { }, WEB_SITE( + new long[] {30, 54, 66, 78, 84, 96}, new Column("WEB_SITE_SK", INT32), new Column("WEB_SITE_ID", STRING), new Column("WEB_REC_START_DATE", DATE), @@ -195,6 +202,7 @@ public String insertPrepareStatement() { }, CUSTOMER_DEMOGRAPHICS( + new long[] {1_920_800, 1_920_800, 1_920_800, 1_920_800, 1_920_800, 1_920_800}, new Column("CD_DEMO_SK", INT32), new Column("CD_GENDER", STRING), new Column("CD_MARITAL_STATUS", STRING), @@ -213,6 +221,7 @@ public String insertPrepareStatement() { }, ITEM( + new long[] {18_000, 300_000, 360_000, 402_000, 462_000, 502_000}, new Column("I_ITEM_SK", INT32), new Column("I_ITEM_ID", STRING), new Column("I_REC_START_DATE", DATE), @@ -246,6 +255,7 @@ public String insertPrepareStatement() { }, WAREHOUSE( + new long[] {5, 20, 22, 25, 27, 30}, new Column("W_WAREHOUSE_SK", INT32), new Column("W_WAREHOUSE_ID", STRING), new Column("W_WAREHOUSE_NAME", STRING), @@ -270,6 +280,7 @@ public String insertPrepareStatement() { }, STORE_RETURNS( + new long[] {287_514, 287_999_764, 863_989_652, 2_879_970_104L, 8_639_952_111L, 28_800_018_820L}, new Column("SR_RETURNED_DATE_SK", INT32), new Column("SR_RETURN_TIME_SK", INT32), new Column("SR_ITEM_SK", INT32), @@ -301,6 +312,7 @@ public String insertPrepareStatement() { }, SHIP_MODE( + new long[] {20, 20, 20, 20, 20, 20}, new Column("SM_SHIP_MODE_SK", INT32), new Column("SM_SHIP_MODE_ID", STRING), new Column("SM_TYPE", STRING), @@ -316,6 +328,7 @@ public String insertPrepareStatement() { }, INCOME_BAND( + new long[] {20, 20, 20, 20, 20, 20}, new Column("IB_INCOME_BAND_SK", INT32), new Column("IB_LOWER_BOUND", INT32), new Column("IB_UPPER_BOUND", INT32) @@ -327,6 +340,7 @@ public String insertPrepareStatement() { }, TIME_DIM( + new long[] {86_400, 86_400, 86_400, 86_400, 86_400, 86_400}, new Column("T_TIME_SK", INT32), new Column("T_TIME_ID", STRING), new Column("T_TIME", INT32), @@ -346,6 +360,7 @@ public String insertPrepareStatement() { }, CATALOG_RETURNS( + new long[] {144_067, 143_996_756, 432_018_033, 1_440_033_112, 4_319_925_093L, 14_400_175_879L}, new Column("CR_RETURNED_DATE_SK", INT32), new Column("CR_RETURNED_TIME_SK", INT32), new Column("CR_ITEM_SK", INT32), @@ -385,6 +400,7 @@ public String insertPrepareStatement() { }, REASON( + new long[] {35, 65, 67, 70, 72, 75}, new Column("R_REASON_SK", INT32), new Column("R_REASON_ID", STRING), new Column("R_REASON_DESC", STRING) @@ -396,6 +412,7 @@ public String insertPrepareStatement() { }, STORE( + new long[] {12, 1_002, 1_350, 1_500, 1_704, 1_902}, new Column("S_STORE_SK", INT32), new Column("S_STORE_ID", STRING), new Column("S_REC_START_DATE", DATE), @@ -438,6 +455,7 @@ public String insertPrepareStatement() { }, INVENTORY( + new long[] {11_745_000, 783_000_000, 1_033_560_000, 1_311_525_000, 1_627_857_000, 1_965_337_830}, new Column("INV_DATE_SK", INT32), new Column("INV_ITEM_SK", INT32), new Column("INV_WAREHOUSE_SK", INT32), @@ -450,6 +468,7 @@ public String insertPrepareStatement() { }, WEB_SALES( + new long[] {719_384, 720_000_376, 2_159_968_881L, 7_199_963_324L, 21_600_036_511L, 71_999_670_164L}, new Column("WS_SOLD_DATE_SK", INT32), new Column("WS_SOLD_TIME_SK", INT32), new Column("WS_SHIP_DATE_SK", INT32), @@ -498,6 +517,7 @@ public String insertPrepareStatement() { }, DATE_DIM( + new long[] {73_049, 73_049, 73_049, 73_049, 73_049, 73_049}, new Column("d_date_sk", INT32), new Column("d_date_id", STRING), new Column("d_date", DATE), @@ -538,6 +558,7 @@ public String insertPrepareStatement() { }, CALL_CENTER( + new long[] {6, 42, 48, 54, 60, 60}, new Column("CC_CALL_CENTER_SK", INT32), new Column("CC_CALL_CENTER_ID", STRING), new Column("CC_REC_START_DATE", DATE), @@ -582,6 +603,7 @@ public String insertPrepareStatement() { }, WEB_RETURNS( + new long[] {71_763, 71_997_522, 216_003_761, 720_020_485, 2_160_007_345L, 7_199_904_459L}, new Column("WR_RETURNED_DATE_SK", INT32), new Column("WR_RETURNED_TIME_SK", INT32), new Column("WR_ITEM_SK", INT32), @@ -618,6 +640,7 @@ public String insertPrepareStatement() { }, PROMOTION( + new long[] {300, 1_500, 1_800, 2_000, 2_300, 2_500}, new Column("P_PROMO_SK", INT32), new Column("P_PROMO_ID", STRING), new Column("P_START_DATE_SK", INT32), @@ -648,6 +671,7 @@ public String insertPrepareStatement() { }, CUSTOMER_ADDRESS( + new long[] {50_000, 6_000_000, 15_000_000, 32_500_000, 40_000_000, 50_000_000}, new Column("CA_ADDRESS_SK", INT32), new Column("CA_ADDRESS_ID", STRING), new Column("CA_STREET_NUMBER", STRING), @@ -671,6 +695,7 @@ public String insertPrepareStatement() { }, CATALOG_SALES( + new long[] {1_441_548, 1_439_980_416, 4_320_078_880L, 14_399_964_710L, 43_200_404_822L, 143_999_334_399L}, new Column("CS_SOLD_DATE_SK", INT32), new Column("CS_SOLD_TIME_SK", INT32), new Column("CS_SHIP_DATE_SK", INT32), @@ -718,8 +743,10 @@ public String insertPrepareStatement() { }; private final Column[] columns; + private final long[] tableSizes; - TpcdsTables(Column... columns) { + TpcdsTables(long[] tableSizes, Column... columns) { + this.tableSizes = tableSizes; this.columns = columns; } @@ -754,6 +781,11 @@ public Iterator dataProvider(Path pathToDataset) throws IOException { .iterator(); } + @Override + public long estimatedSize(TpcScaleFactor sf) { + return sf.size(tableSizes); + } + private Object[] csvLineToTableValues(String line) { String[] stringValues = line.split("\\|"); Object[] values = new Object[columns.length]; diff --git a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpch/TpchTables.java b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpch/TpchTables.java index bdd2b60cd64..41caba81b38 100644 --- a/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpch/TpchTables.java +++ b/modules/sql-engine/src/testFixtures/java/org/apache/ignite/internal/sql/engine/util/tpch/TpchTables.java @@ -28,6 +28,7 @@ import java.nio.file.Path; import java.time.LocalDate; import java.util.Iterator; +import org.apache.ignite.internal.sql.engine.util.TpcScaleFactor; import org.apache.ignite.internal.sql.engine.util.TpcTable; import org.apache.ignite.sql.ColumnType; @@ -37,6 +38,7 @@ @SuppressWarnings("NonSerializableFieldInSerializableClass") public enum TpchTables implements TpcTable { LINEITEM( + new long[] {6_001_215, 5_999_989_709L, 18_000_048_306L, 59_999_994_267L, 179_999_978_268L, 599_999_969_200L}, new Column("L_ORDERKEY", INT32), new Column("L_PARTKEY", INT32), new Column("L_SUPPKEY", INT32), @@ -65,6 +67,7 @@ public String insertPrepareStatement() { }, PART( + new long[] {200_000, 200_000_000, 600_000_000, 2_000_000_000, 6_000_000_000L, 20_000_000_000L}, new Column("P_PARTKEY", INT32), new Column("P_NAME", STRING), new Column("P_MFGR", STRING), @@ -85,6 +88,7 @@ public String insertPrepareStatement() { }, SUPPLIER( + new long[] {10_000, 10_000_000, 30_000_000, 100_000_000, 300_000_000, 1_000_000_000}, new Column("S_SUPPKEY", INT32), new Column("S_NAME", STRING), new Column("S_ADDRESS", STRING), @@ -102,6 +106,7 @@ public String insertPrepareStatement() { }, PARTSUPP( + new long[] {800_000, 800_000_000, 2_400_000_000L, 8_000_000_000L, 24_000_000_000L, 80_000_000_000L}, new Column("PS_PARTKEY", INT32), new Column("PS_SUPPKEY", INT32), new Column("PS_AVAILQTY", INT32), @@ -117,6 +122,7 @@ public String insertPrepareStatement() { }, NATION( + new long[] {25, 25, 25, 25, 25, 25}, new Column("N_NATIONKEY", INT32), new Column("N_NAME", STRING), new Column("N_REGIONKEY", INT32), @@ -130,6 +136,7 @@ public String insertPrepareStatement() { }, REGION( + new long[] {5, 5, 5, 5, 5, 5}, new Column("R_REGIONKEY", INT32), new Column("R_NAME", STRING), new Column("R_COMMENT", STRING) @@ -142,6 +149,7 @@ public String insertPrepareStatement() { }, ORDERS( + new long[] {1_500_000, 1_500_000_000, 4_500_000_000L, 15_000_000_000L, 45_000_000_000L, 150_000_000_000L}, new Column("O_ORDERKEY", INT32), new Column("O_CUSTKEY", INT32), new Column("O_ORDERSTATUS", STRING), @@ -162,6 +170,7 @@ public String insertPrepareStatement() { }, CUSTOMER( + new long[] {150_000, 150_000_000, 450_000_000L, 1_500_000_000L, 4_500_000_000L, 15_000_000_000L}, new Column("C_CUSTKEY", INT32), new Column("C_NAME", STRING), new Column("C_ADDRESS", STRING), @@ -181,8 +190,10 @@ public String insertPrepareStatement() { }; private final Column[] columns; + private final long[] tableSizes; - TpchTables(Column... columns) { + TpchTables(long[] tableSizes, Column... columns) { + this.tableSizes = tableSizes; this.columns = columns; } @@ -214,6 +225,11 @@ public Iterator dataProvider(Path pathToDataset) throws IOException { .iterator(); } + @Override + public long estimatedSize(TpcScaleFactor sf) { + return sf.size(tableSizes); + } + private Object[] csvLineToTableValues(String line) { String[] stringValues = line.split("\\|"); Object[] values = new Object[columns.length]; diff --git a/modules/sql-engine/src/testFixtures/resources/tpcds/query64.sql b/modules/sql-engine/src/testFixtures/resources/tpcds/query64.sql index e353b930dd1..5279bd6b259 100644 --- a/modules/sql-engine/src/testFixtures/resources/tpcds/query64.sql +++ b/modules/sql-engine/src/testFixtures/resources/tpcds/query64.sql @@ -84,7 +84,9 @@ group by i_product_name ,d2.d_year ,d3.d_year ) -select cs1.product_name +select + /*+ NO_INDEX, DISABLE_RULE('MergeJoinConverter') */ + cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number diff --git a/modules/sql-engine/src/testFixtures/resources/tpch/q5.sql b/modules/sql-engine/src/testFixtures/resources/tpch/q5.sql index 2a07e3a5de5..1eff131086c 100644 --- a/modules/sql-engine/src/testFixtures/resources/tpch/q5.sql +++ b/modules/sql-engine/src/testFixtures/resources/tpch/q5.sql @@ -2,6 +2,7 @@ -- noinspection SqlNoDataSourceInspectionForFile SELECT + /*+ NO_INDEX, DISABLE_RULE('MergeJoinConverter') */ n_name, sum(l_extendedprice * (1 - l_discount)) AS revenue FROM diff --git a/modules/sql-engine/src/testFixtures/resources/tpch/q7.sql b/modules/sql-engine/src/testFixtures/resources/tpch/q7.sql index a98318a1f36..04871640380 100644 --- a/modules/sql-engine/src/testFixtures/resources/tpch/q7.sql +++ b/modules/sql-engine/src/testFixtures/resources/tpch/q7.sql @@ -2,6 +2,7 @@ -- noinspection SqlNoDataSourceInspectionForFile SELECT + /*+ NO_INDEX, DISABLE_RULE('MergeJoinConverter') */ supp_nation, cust_nation, l_year, diff --git a/modules/sql-engine/src/testFixtures/resources/tpch/q8.sql b/modules/sql-engine/src/testFixtures/resources/tpch/q8.sql index 617d098c1ab..007b99ab84e 100644 --- a/modules/sql-engine/src/testFixtures/resources/tpch/q8.sql +++ b/modules/sql-engine/src/testFixtures/resources/tpch/q8.sql @@ -2,6 +2,7 @@ -- noinspection SqlNoDataSourceInspectionForFile SELECT + /*+ NO_INDEX, DISABLE_RULE('MergeJoinConverter') */ o_year, sum(CASE WHEN nation = 'BRAZIL' diff --git a/modules/sql-engine/src/testFixtures/resources/tpch/q9.sql b/modules/sql-engine/src/testFixtures/resources/tpch/q9.sql index 0fdc07f1bf4..48b182476e2 100644 --- a/modules/sql-engine/src/testFixtures/resources/tpch/q9.sql +++ b/modules/sql-engine/src/testFixtures/resources/tpch/q9.sql @@ -2,6 +2,7 @@ -- noinspection SqlNoDataSourceInspectionForFile SELECT + /*+ NO_INDEX, DISABLE_RULE('MergeJoinConverter') */ nation, o_year, sum(amount) AS sum_profit