diff --git a/LICENSE b/LICENSE index 549d757478..c1c8f6f847 100644 --- a/LICENSE +++ b/LICENSE @@ -237,6 +237,10 @@ The following file are provided under the Apache 2.0 License. linkis-web/public/favicon.ico linkis-engineconn-plugins/seatunnel/src/main/java/org/apache/seatunnel/* linkis-commons/linkis-storage/src/test/resources/scritpis-test.sql + linkis-engineconn-plugins/hbase/hbase-shims-1.2.0/src/main/resources/hbase-ruby/* + linkis-engineconn-plugins/hbase/hbase-shims-1.4.3/src/main/resources/hbase-ruby/* + linkis-engineconn-plugins/hbase/hbase-shims-2.2.6/src/main/resources/hbase-ruby/* + linkis-engineconn-plugins/hbase/hbase-shims-2.5.3/src/main/resources/hbase-ruby/* The files: .mvn/wrapper/MavenWrapperDownloader.java diff --git a/docs/configuration/spark.md b/docs/configuration/spark.md index f40c76b43d..ed070e2ac4 100644 --- a/docs/configuration/spark.md +++ b/docs/configuration/spark.md @@ -29,7 +29,10 @@ |spark|wds.linkis.spark.engine.scala.replace_package_header.enable| true |spark.engine.scala.replace_package_header.enable| Use spark yarn cluster mode,need to set label "engingeConnRuntimeMode": "yarnCluster",and need to upload the dependence of the spark to 'linkis.spark.yarn.cluster.jar'(the default value is 'hdfs:///spark/cluster') -spark dependencies include jars and configuration files，For example: '/appcom/Install/linkis/lib/linkis-engineconn-plugins/spark/dist/3.2.1/lib/*.jar','/appcom/Install/linkis/conf/*'' +spark dependencies include jars and configuration files，For example: '/appcom/Install/linkis/lib/linkis-engineconn-plugins/spark/dist/3.2.1/lib/*.jar','/appcom/Install/linkis/conf/*' + +Precautions for using yarnCluster: +Eureka url if 127.0.0.1 should be changed to the real host, such as "127.0.0.1:20303/eureka/" should be changed to "wds001:20303/eureka/" The spark-excel package may cause class conflicts,need to download separately,put it in spark lib wget https://repo1.maven.org/maven2/com/crealytics/spark-excel-2.12.17-3.2.2_2.12/3.2.2_0.18.1/spark-excel-2.12.17-3.2.2_2.12-3.2.2_0.18.1.jar diff --git a/docs/errorcode/linkis-instance-label-errorcode.md b/docs/errorcode/linkis-instance-label-errorcode.md index 7f38e00a1e..5aa7432438 100644 --- a/docs/errorcode/linkis-instance-label-errorcode.md +++ b/docs/errorcode/linkis-instance-label-errorcode.md @@ -1,7 +1,7 @@ ## linkis-instance-label errorcode | 模块名(服务名) | 错误码 | 描述 | Exception Class| -| -------- | -------- | ----- |-----| +| | -------- | ----- |-----| |linkis-instance-label |14100|Failed to insert service instance(插入服务实例失败)|LinkisInstanceLabelErrorCodeSummary| |linkis-instance-label |14100|Only admin can view all instances(只有管理员才能查看所有实例).|LinkisInstanceLabelErrorCodeSummary| |linkis-instance-label |14100|Only admin can modify instance label(只有管理员才能修改标签).|LinkisInstanceLabelErrorCodeSummary| diff --git a/licenses/LICENSE-hbase-shell-ruby.txt b/licenses/LICENSE-hbase-shell-ruby.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/licenses/LICENSE-hbase-shell-ruby.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/io/FsPath.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/io/FsPath.java index 7908e5a506..a1c0839b2f 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/io/FsPath.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/io/FsPath.java @@ -23,6 +23,8 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.FileSystems; +import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; @@ -168,6 +170,10 @@ public File toFile() { return new File(uri); } + public Path toPath() { + return FileSystems.getDefault().getPath(uri.toString()); + } + public String getUriString() { return uri.toString(); } diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java index d23f4a0867..0ecb3dc2a5 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java @@ -213,7 +213,6 @@ private static long parseByteString(String str, ByteUnit unit) { } else { throw new NumberFormatException("Failed to parse byte string: " + str); } - suffix = suffix.toLowerCase(); // Check for invalid suffixes if (suffix != null && !byteSuffixes.containsKey(suffix)) { throw new NumberFormatException("Invalid suffix: \"" + suffix + "\""); @@ -297,6 +296,18 @@ public static long byteStringAsGb(String str) { return parseByteString(str, ByteUnit.GiB); } + /** + * Convert a passed byte string (e.g. -50b, -100k, or -250m) to gibibytes for internal use. + * + *

If no suffix is provided, the passed number is assumed to be in gibibytes. + */ + public static long negativeByteStringAsGb(String str) { + if (str.startsWith("-")) { + return Math.negateExact(parseByteString(str.substring(1), ByteUnit.GiB)); + } + return parseByteString(str, ByteUnit.GiB); + } + /** * Returns a byte array with the buffer's contents, trying to avoid copying the data if possible. */ diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java new file mode 100644 index 0000000000..a367b38b80 --- /dev/null +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import org.apache.linkis.common.io.FsPath; + +import java.io.File; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ResultSetUtils { + + // Sort in ASC order by numx in the result set _numx.dolphin file name + public static Comparator getResultSetFileComparatorOrderByNameNum() { + + Comparator comparator = + (o1, o2) -> { + // get the num of file name + String regx = "\\d+"; + + String[] res1 = o1.getPath().split(File.separator); + String fileName1 = res1[res1.length - 1]; + Matcher matcher1 = Pattern.compile(regx).matcher(fileName1); + int num1 = matcher1.find() ? Integer.parseInt(matcher1.group()) : Integer.MAX_VALUE; + + String[] res2 = o2.getPath().split(File.separator); + String fileName2 = res2[res2.length - 1]; + Matcher matcher2 = Pattern.compile(regx).matcher(fileName2); + int num2 = matcher2.find() ? Integer.parseInt(matcher2.group()) : Integer.MAX_VALUE; + + return num1 - num2; + }; + return comparator; + } + + public static void sortByNameNum(List fsPathList) { + Collections.sort(fsPathList, getResultSetFileComparatorOrderByNameNum()); + } +} diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java index b891f99d15..615472474d 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java @@ -28,7 +28,10 @@ import java.util.Iterator; import java.util.Map; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -49,6 +52,9 @@ public class VariableOperationUtils { private static final String[] CYCLES = new String[] {CYCLE_YEAR, CYCLE_MONTH, CYCLE_DAY, CYCLE_HOUR, CYCLE_MINUTE, CYCLE_SECOND}; + private static final ObjectMapper mapper = + JsonMapper.builder().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS).build(); + /** * yyyy-MM-dd HH:mm:ss * @@ -78,30 +84,44 @@ public static ZonedDateTime toZonedDateTime(Date date) { * @param str * @return */ + @Deprecated public static String replaces(ZonedDateTime dateTime, String str) throws VariableOperationFailedException { - return replaces(dateTime, str, true); + try { + JsonNode rootNode = mapper.readTree(str); + if (rootNode.isArray() || rootNode.isObject()) { + replaceJson(dateTime, rootNode); + return rootNode.toString(); + } + } catch (Exception e) { + return replace(dateTime, str); + } + return replace(dateTime, str); } /** * json support variable operation * + * @param codeType * @param dateTime * @param str - * @param format * @return */ - public static String replaces(ZonedDateTime dateTime, String str, boolean format) + public static String replaces(String codeType, ZonedDateTime dateTime, String str) throws VariableOperationFailedException { - try { - JsonNode rootNode = JsonUtils.jackson().readTree(str); - if (rootNode.isArray() || rootNode.isObject()) { - replaceJson(dateTime, rootNode); - return rootNode.toString(); + String languageType = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType, ""); + if (languageType.equals(CodeAndRunTypeUtils.LANGUAGE_TYPE_JSON())) { + try { + JsonNode rootNode = mapper.readTree(str); + if (rootNode.isArray() || rootNode.isObject()) { + replaceJson(dateTime, rootNode); + return rootNode.toString(); + } + } catch (Exception e) { + return replace(dateTime, str); } - } catch (Exception e) { - return replace(dateTime, str); } + return replace(dateTime, str); } @@ -197,8 +217,7 @@ private static void replaceJson(ZonedDateTime dateTime, JsonNode object) } else if (temp.isObject()) { replaceJson(dateTime, temp); } else { - arrayNode.remove(i); - arrayNode.insert(i, replace(dateTime, temp.toString())); + arrayNode.set(i, replace(dateTime, temp.toString())); } } } else if (object.isObject()) { diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala index 8fcb4af737..f9e4718472 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala @@ -20,11 +20,17 @@ package org.apache.linkis.common class ServiceInstance { private var applicationName: String = _ private var instance: String = _ + private var registryTimestamp: Long = _ def setApplicationName(applicationName: String): Unit = this.applicationName = applicationName def getApplicationName: String = applicationName def setInstance(instance: String): Unit = this.instance = instance def getInstance: String = instance + def setRegistryTimestamp(registryTimestamp: Long): Unit = this.registryTimestamp = + registryTimestamp + + def getRegistryTimestamp: Long = registryTimestamp + override def equals(other: Any): Boolean = other match { case that: ServiceInstance => applicationName == that.applicationName && @@ -42,7 +48,9 @@ class ServiceInstance { .foldLeft(0)((a, b) => 31 * a + b) } - override def toString: String = s"ServiceInstance($applicationName, $instance)" + override def toString: String = + s"ServiceInstance($applicationName, $instance, $registryTimestamp)" + } object ServiceInstance { @@ -54,6 +62,14 @@ object ServiceInstance { serviceInstance } + def apply(applicationName: String, instance: String, registryTimestamp: Long): ServiceInstance = { + val serviceInstance = new ServiceInstance + serviceInstance.setApplicationName(applicationName) + serviceInstance.setInstance(instance) + serviceInstance.setRegistryTimestamp(registryTimestamp) + serviceInstance + } + def unapply(serviceInstance: ServiceInstance): Option[(String, String)] = if (serviceInstance != null) { Some(serviceInstance.applicationName, serviceInstance.instance) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala index 14febab63a..9bfa053b77 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala @@ -232,19 +232,20 @@ private[conf] object BDPConfiguration extends Logging { private[common] def formatValue[T](defaultValue: T, value: Option[String]): Option[T] = { if (value.isEmpty || value.exists(StringUtils.isEmpty)) return Option(defaultValue) + val trimValue = value.map(_.trim) val formattedValue = defaultValue match { - case _: String => value - case _: Byte => value.map(_.toByte) - case _: Short => value.map(_.toShort) - case _: Char => value.map(_.toCharArray.apply(0)) - case _: Int => value.map(_.toInt) - case _: Long => value.map(_.toLong) - case _: Float => value.map(_.toFloat) - case _: Double => value.map(_.toDouble) - case _: Boolean => value.map(_.toBoolean) - case _: TimeType => value.map(new TimeType(_)) - case _: ByteType => value.map(new ByteType(_)) - case null => value + case _: String => trimValue + case _: Byte => trimValue.map(_.toByte) + case _: Short => trimValue.map(_.toShort) + case _: Char => trimValue.map(_.toCharArray.apply(0)) + case _: Int => trimValue.map(_.toInt) + case _: Long => trimValue.map(_.toLong) + case _: Float => trimValue.map(_.toFloat) + case _: Double => trimValue.map(_.toDouble) + case _: Boolean => trimValue.map(_.toBoolean) + case _: TimeType => trimValue.map(new TimeType(_)) + case _: ByteType => trimValue.map(new ByteType(_)) + case null => trimValue } formattedValue.asInstanceOf[Option[T]] } diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/log/LogUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/log/LogUtils.scala index 77c82f3883..e558e765be 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/log/LogUtils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/log/LogUtils.scala @@ -33,7 +33,7 @@ object LogUtils { } def generateERROR(rawLog: String): String = { - getTimeFormat + " " + "ERROR" + " " + rawLog + getTimeFormat + " " + ERROR_STR + " " + rawLog } def generateWarn(rawLog: String): String = { @@ -52,4 +52,6 @@ object LogUtils { getTimeFormat + " " + "SYSTEM-WARN" + " " + rawLog } + val ERROR_STR = "ERROR" + } diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/CodeAndRunTypeUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/CodeAndRunTypeUtils.scala index 3870fe6e58..917ac53261 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/CodeAndRunTypeUtils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/CodeAndRunTypeUtils.scala @@ -21,6 +21,8 @@ import org.apache.linkis.common.conf.CommonVars import org.apache.commons.lang3.StringUtils +import scala.collection.mutable + object CodeAndRunTypeUtils { private val CONF_LOCK = new Object() @@ -101,7 +103,14 @@ object CodeAndRunTypeUtils { def getLanguageTypeAndCodeTypeRelationMap: Map[String, String] = { val codeTypeAndRunTypeRelationMap = getCodeTypeAndLanguageTypeRelationMap if (codeTypeAndRunTypeRelationMap.isEmpty) Map() - else codeTypeAndRunTypeRelationMap.flatMap(x => x._2.map(y => (y, x._1))) + else { +// codeTypeAndRunTypeRelationMap.flatMap(x => x._2.map(y => (y, x._1))) + val map = mutable.Map[String, String]() + codeTypeAndRunTypeRelationMap.foreach(kv => { + kv._2.foreach(v => map.put(v, kv._1)) + }) + map.toMap + } } def getLanguageTypeByCodeType(codeType: String, defaultLanguageType: String = ""): String = { diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala index 30bdeb4b14..6c5bd7cf3c 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala @@ -143,9 +143,10 @@ object VariableUtils extends Logging { } initAllDateVars(run_date, nameAndType) val codeOperation = parserVar(code, nameAndType) - parserDate(codeOperation, run_date) + parserDate(codeType, codeOperation, run_date) } + @deprecated private def parserDate(code: String, run_date: CustomDateType): String = { if (Configuration.VARIABLE_OPERATION) { val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate) @@ -155,6 +156,15 @@ object VariableUtils extends Logging { } } + private def parserDate(codeType: String, code: String, run_date: CustomDateType): String = { + if (Configuration.VARIABLE_OPERATION) { + val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate) + VariableOperationUtils.replaces(codeType, zonedDateTime, code) + } else { + code + } + } + private def initAllDateVars( run_date: CustomDateType, nameAndType: mutable.Map[String, variable.VariableType] @@ -337,7 +347,7 @@ object VariableUtils extends Logging { * * @param code * :code - * @param codeType + * @param languageType * :SQL,PYTHON * @return */ @@ -346,27 +356,37 @@ object VariableUtils extends Logging { var varString: String = null var errString: String = null + var rightVarString: String = null languageType match { case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL => varString = """\s*--@set\s*.+\s*""" + rightVarString = """^\s*--@set\s*.+\s*""" errString = """\s*--@.*""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL => varString = """\s*#@set\s*.+\s*""" + rightVarString = """^\s*#@set\s*.+\s*""" errString = """\s*#@""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA => varString = """\s*//@set\s*.+\s*""" + rightVarString = """^\s*//@set\s*.+\s*""" errString = """\s*//@.+""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_JAVA => varString = """\s*!!@set\s*.+\s*""" + rightVarString = """^\s*!!@set\s*.+\s*""" case _ => return nameAndValue } val customRegex = varString.r.unanchored + val customRightRegex = rightVarString.r.unanchored val errRegex = errString.r.unanchored code.split("\n").foreach { str => { + + if (customRightRegex.unapplySeq(str).size < customRegex.unapplySeq(str).size) { + logger.warn(s"code:$str is wrong custom variable format!!!") + } str match { case customRegex() => val clearStr = if (str.endsWith(";")) str.substring(0, str.length - 1) else str diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java index 7b90f053ca..f548d89d46 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java @@ -17,31 +17,159 @@ package org.apache.linkis.common.utils; +import java.util.function.Function; + +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - class ByteTimeUtilsTest { + private static final ImmutableMap> opFunction = + ImmutableMap.>builder() + .put("byteStringAsBytes", tar -> ByteTimeUtils.byteStringAsBytes(tar)) + .put("byteStringAsKb", tar -> ByteTimeUtils.byteStringAsKb(tar)) + .put("byteStringAsMb", tar -> ByteTimeUtils.byteStringAsMb(tar)) + .put("byteStringAsGb", tar -> ByteTimeUtils.byteStringAsGb(tar)) + .build(); + + private static final ImmutableMap convertToByte = + ImmutableMap.builder() + .put("1", 1l) + .put("1b", 1l) + .put("1B", 1l) + .put("1k", 1024l) + .put("1K", 1024l) + .put("1kb", 1024l) + .put("1Kb", 1024l) + .put("1kB", 1024l) + .put("1KB", 1024l) + .put("1m", 1024l * 1024l) + .put("1M", 1024l * 1024l) + .put("1mb", 1024l * 1024l) + .put("1Mb", 1024l * 1024l) + .put("1mB", 1024l * 1024l) + .put("1MB", 1024l * 1024l) + .put("1g", 1024l * 1024l * 1024l) + .put("1G", 1024l * 1024l * 1024l) + .put("1gb", 1024l * 1024l * 1024l) + .put("1gB", 1024l * 1024l * 1024l) + .put("1Gb", 1024l * 1024l * 1024l) + .put("1GB", 1024l * 1024l * 1024l) + .put("1t", 1024l * 1024l * 1024l * 1024l) + .put("1T", 1024l * 1024l * 1024l * 1024l) + .put("1tb", 1024l * 1024l * 1024l * 1024l) + .put("1Tb", 1024l * 1024l * 1024l * 1024l) + .put("1tB", 1024l * 1024l * 1024l * 1024l) + .put("1TB", 1024l * 1024l * 1024l * 1024l) + .put("1p", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1P", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1Pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pB", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1PB", 1024l * 1024l * 1024l * 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToKB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024b", 1l) + .put("1024B", 1l) + .put("1k", 1l) + .put("1K", 1l) + .put("1kb", 1l) + .put("1Kb", 1l) + .put("1kB", 1l) + .put("1KB", 1l) + .put("1m", 1024l) + .put("1M", 1024l) + .put("1mb", 1024l) + .put("1Mb", 1024l) + .put("1mB", 1024l) + .put("1MB", 1024l) + .put("1g", 1024l * 1024l) + .put("1G", 1024l * 1024l) + .put("1gb", 1024l * 1024l) + .put("1gB", 1024l * 1024l) + .put("1Gb", 1024l * 1024l) + .put("1GB", 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToMB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024k", 1l) + .put("1024K", 1l) + .put("1024kb", 1l) + .put("1024Kb", 1l) + .put("1024kB", 1l) + .put("1024KB", 1l) + .put("1m", 1l) + .put("1M", 1l) + .put("1mb", 1l) + .put("1Mb", 1l) + .put("1mB", 1l) + .put("1MB", 1l) + .put("1g", 1024l) + .put("1G", 1024l) + .put("1gb", 1024l) + .put("1gB", 1024l) + .put("1Gb", 1024l) + .put("1GB", 1024l) + .build(); + + private static final ImmutableMap convertToGB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024m", 1l) + .put("1024M", 1l) + .put("1024mb", 1l) + .put("1024Mb", 1l) + .put("1024mB", 1l) + .put("1024MB", 1l) + .put("1g", 1l) + .put("1G", 1l) + .put("1gb", 1l) + .put("1gB", 1l) + .put("1Gb", 1l) + .put("1GB", 1l) + .put("1t", 1024l) + .put("1T", 1024l) + .put("1tb", 1024l) + .put("1Tb", 1024l) + .put("1tB", 1024l) + .put("1TB", 1024l) + .build(); + @Test - void byteStringAsBytes() {} + void byteStringAsBytes() { + convertToByte.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsBytes").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsBytes").apply("1A")); + } @Test - void byteStringAsKb() {} + void byteStringAsKb() { + convertToKB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsKb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsKb").apply("1a")); + } @Test - void byteStringAsMb() {} + void byteStringAsMb() { + convertToMB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsMb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsMb").apply("1c")); + } @Test void byteStringAsGb() { - Long res = ByteTimeUtils.byteStringAsGb("1G"); - Assertions.assertEquals(res, 1); - + convertToGB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsGb").apply(k), v)); Assertions.assertThrows( - NullPointerException.class, - () -> { - ByteTimeUtils.byteStringAsGb("512"); - }); + IllegalArgumentException.class, () -> opFunction.get("byteStringAsGb").apply("1C")); } } diff --git a/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala b/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala index c0d4ad1d61..e7a105497c 100644 --- a/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala +++ b/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala @@ -22,6 +22,8 @@ import org.apache.linkis.common.variable.DateTypeUtils.{getCurHour, getToday} import java.util +import scala.collection.mutable + import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -63,4 +65,19 @@ class VariableUtilsTest { assertEquals(VariableUtils.replace(sql, "sql", varMap), resSql) } + @Test + def testGetCustomVar: Unit = { + var scalaCode = "" + + "-------@set globalpara=60--------\n" + + "--@set globalpara2=66\n" + + "select ${globalpara} as globalpara,\n" + + "-- ${globalpara1} as globalpara1, \n" + + "${globalpara2} as globalpara2;\n" + var pythonCode = "" + + val nameAndValue: mutable.Map[String, String] = + VariableUtils.getCustomVar(scalaCode, CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL); + assertEquals(nameAndValue.size, 2) + } + } diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala index d901bb902e..ec61ebd66d 100644 --- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala +++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala @@ -143,23 +143,31 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String val req = prepareReq(action) val startTime = System.currentTimeMillis val response = executeRequest(req, Some(waitTime).filter(_ > 0)) + val taken = System.currentTimeMillis - startTime + attempts.add(taken) + val costTime = ByteTimeUtils.msDurationToString(taken) + logger.info( + s"invoke ${req.getURI} get status ${response.getStatusLine.getStatusCode} taken: ${costTime}." + ) if (response.getStatusLine.getStatusCode == 401) { tryLogin(action, getRequestUrl(action), true) - logger.info("The user is not logged in, please log in first, you can set a retry") val msg = Utils.tryCatch(EntityUtils.toString(response.getEntity)) { t => logger.warn("failed to parse entity", t) "" } IOUtils.closeQuietly(response) - throw new HttpClientRetryException( - "The user is not logged in, please log in first, you can set a retry, message: " + msg - ) + if (attempts.size() <= 1) { + logger.info("The user is not logged in, default retry once") + addAttempt() + } else { + logger.info("The user is not logged in, you can set a retry") + throw new HttpClientRetryException( + "The user is not logged in, please log in first, you can set a retry, message: " + msg + ) + } + } else { + response } - val taken = System.currentTimeMillis - startTime - attempts.add(taken) - val costTime = ByteTimeUtils.msDurationToString(taken) - logger.info(s"invoke ${req.getURI} taken: ${costTime}.") - response } val response = diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala index 26170a4577..b1fc579f3c 100644 --- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala +++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala @@ -17,9 +17,9 @@ package org.apache.linkis.httpclient.config +import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{DefaultRetryHandler, RetryHandler} import org.apache.linkis.httpclient.authentication.AuthenticationStrategy -import org.apache.linkis.httpclient.exception.HttpClientRetryException import org.apache.linkis.httpclient.loadbalancer.LoadBalancerStrategy import scala.concurrent.duration.TimeUnit @@ -39,11 +39,10 @@ class ClientConfigBuilder protected () { protected var readTimeout: Long = _ protected var maxConnection: Int = _ protected var retryEnabled: Boolean = true - protected var retryHandler: RetryHandler = buildDefaultRetryHandler() - def buildDefaultRetryHandler(): RetryHandler = { - retryHandler = new DefaultRetryHandler - retryHandler.addRetryException(classOf[HttpClientRetryException]) + protected var retryHandler: RetryHandler = { + val retryHandler = new DefaultRetryHandler + retryHandler.addRetryException(classOf[LinkisRetryException]) retryHandler } diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala index 582568e626..3c6a25a343 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala @@ -207,4 +207,7 @@ object ServerConfiguration extends Logging { val LINKIS_SERVER_SESSION_PROXY_TICKETID_KEY = CommonVars("wds.linkis.session.proxy.user.ticket.key", "linkis_user_session_proxy_ticket_id_v1") + val LINKIS_SERVER_ENTRANCE_HEADER_KEY = + CommonVars("linkis.server.entrance.header.key", "jobInstanceKey") + } diff --git a/linkis-commons/linkis-mybatis/pom.xml b/linkis-commons/linkis-mybatis/pom.xml index b87a1ed66a..1481f2008d 100644 --- a/linkis-commons/linkis-mybatis/pom.xml +++ b/linkis-commons/linkis-mybatis/pom.xml @@ -37,6 +37,32 @@ com.baomidou mybatis-plus-boot-starter + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-autoconfigure + + + org.springframework + spring-beans + + + org.springframework + spring-jdbc + + + com.zaxxer + HikariCP + + + + + org.springframework + spring-jdbc com.github.pagehelper diff --git a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java index 42661f8283..a90a1eb3b7 100644 --- a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java +++ b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java @@ -69,6 +69,8 @@ public interface TaskConstant { String TICKET_ID = "ticketId"; String ENGINE_CONN_TASK_ID = "engineConnTaskId"; String ENGINE_CONN_SUBMIT_TIME = "engineConnSubmitTime"; + String FAILOVER_FLAG = "failoverFlag"; + String DEBUG_ENBALE = "debug.enable"; String PARAMS_DATA_SOURCE = "dataSources"; diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/engine/JobInstance.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/engine/JobInstance.scala new file mode 100644 index 0000000000..5e2eb10a59 --- /dev/null +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/engine/JobInstance.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.protocol.engine + +case class JobInstance( + status: String, + instances: String, + jobReqId: String, + createTimestamp: Long, + instanceRegistryTimestamp: Long +) diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala index 3b94bbdc14..3affc351d9 100644 --- a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/TaskUtils.scala @@ -45,6 +45,14 @@ object TaskUtils { } } else params.put(key, waitToAdd) + private def clearMap(params: util.Map[String, AnyRef], key: String): Unit = + if (params != null && params.containsKey(key)) { + params.get(key) match { + case map: util.Map[String, AnyRef] => map.clear() + case _ => params.put(key, new util.HashMap[String, AnyRef]()) + } + } + private def getConfigurationMap( params: util.Map[String, AnyRef], key: String @@ -84,17 +92,35 @@ object TaskUtils { def addStartupMap(params: util.Map[String, AnyRef], startupMap: util.Map[String, AnyRef]): Unit = addConfigurationMap(params, startupMap, TaskConstant.PARAMS_CONFIGURATION_STARTUP) + def clearStartupMap(params: util.Map[String, AnyRef]): Unit = { + val configurationMap = getMap(params, TaskConstant.PARAMS_CONFIGURATION) + if (!configurationMap.isEmpty) { + clearMap(configurationMap, TaskConstant.PARAMS_CONFIGURATION_STARTUP) + } + } + def addRuntimeMap(params: util.Map[String, AnyRef], runtimeMap: util.Map[String, AnyRef]): Unit = addConfigurationMap(params, runtimeMap, TaskConstant.PARAMS_CONFIGURATION_RUNTIME) def addSpecialMap(params: util.Map[String, AnyRef], specialMap: util.Map[String, AnyRef]): Unit = addConfigurationMap(params, specialMap, TaskConstant.PARAMS_CONFIGURATION_SPECIAL) - // tdoo + // todo def getLabelsMap(params: util.Map[String, AnyRef]): util.Map[String, AnyRef] = getMap(params, TaskConstant.LABELS) def addLabelsMap(params: util.Map[String, AnyRef], labels: util.Map[String, AnyRef]): Unit = addMap(params, labels, TaskConstant.LABELS) + def isWithDebugInfo(params: util.Map[String, AnyRef]): Boolean = { + val debug = getConfigurationMap(params, TaskConstant.PARAMS_CONFIGURATION_STARTUP).get( + TaskConstant.DEBUG_ENBALE + ) + if (debug != null && "true".equals(debug.toString)) { + true + } else { + false + } + } + } diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala index 95c7a81873..ad30484c46 100644 --- a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala @@ -23,7 +23,7 @@ object ZuulEntranceUtils { private val INSTANCE_SPLIT_TOKEN = "_" - private val EXEC_ID = "exec_id" + val EXEC_ID = "exec_id" private val SPLIT_LEN = 3 diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala index 32f12da273..bbe2d4acd3 100644 --- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala +++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala @@ -65,14 +65,6 @@ object RPCConfiguration { "cs,contextservice,data-source-manager,metadataQuery,metadatamanager,query,jobhistory,application,configuration,filesystem,udf,variable,microservice,errorcode,bml,datasource,basedata-manager" ).getValue.split(",") - val METADATAQUERY_SERVICE_APPLICATION_NAME: CommonVars[String] = - CommonVars("wds.linkis.gateway.conf.publicservice.name", "linkis-ps-metadataquery") - - val METADATAQUERY_SERVICE_LIST: Array[String] = CommonVars( - "wds.linkis.gateway.conf.metadataquery.list", - "metadatamanager,metadataquery" - ).getValue.split(",") - val LINKIS_MANAGER_SERVICE_NAME: CommonVars[String] = CommonVars("wds.linkis.gateway.conf.linkismanager.name", "linkis-cg-linkismanager") @@ -80,6 +72,16 @@ object RPCConfiguration { CommonVars("wds.linkis.gateway.conf.linkismanager.list", "linkisManager,engineplugin").getValue .split(",") + val LINKIS_DATASOURCE_SERVICE_NAME: CommonVars[String] = + CommonVars("linkis.gateway.conf.linkisdatasource.name", "linkis-ps-datasource") + + val LINKIS_DATASOURCE_SERVICE_LIST: Array[String] = + CommonVars( + "linkis.gateway.conf.linkisdatasource.list", + "data-source-manager,metadataquery,datasource,metadataQuery,metadatamanager" + ).getValue + .split(",") + val BDP_RPC_INSTANCE_ALIAS_SERVICE_REFRESH_INTERVAL: CommonVars[TimeType] = CommonVars("wds.linkis.rpc.instancealias.refresh.interval", new TimeType("3s")) diff --git a/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala b/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala index 5aaadd133c..ed1d39ce8c 100644 --- a/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala +++ b/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/conf/RPCConfigurationTest.scala @@ -37,7 +37,6 @@ class RPCConfigurationTest { val enablepublicservice = RPCConfiguration.ENABLE_PUBLIC_SERVICE.getValue val publicserviceapplicationname = RPCConfiguration.PUBLIC_SERVICE_APPLICATION_NAME.getValue val publicservicelist = RPCConfiguration.PUBLIC_SERVICE_LIST - val metadataqueryservicelist = RPCConfiguration.METADATAQUERY_SERVICE_LIST Assertions.assertTrue(25 == bdprpcbroadcastthreadsize.intValue()) Assertions.assertTrue(400 == bdprpcreceiverasynconsumerthreadmax.intValue()) @@ -48,7 +47,6 @@ class RPCConfigurationTest { Assertions.assertTrue(enablepublicservice) Assertions.assertEquals("linkis-ps-publicservice", publicserviceapplicationname) Assertions.assertTrue(publicservicelist.size > 0) - Assertions.assertTrue(metadataqueryservicelist.size > 0) } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala index 6e9ecbd26f..b123682b56 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala @@ -23,11 +23,18 @@ abstract class AbstractGroup extends Group { private var _status: GroupStatus = _ private var maxRunningJobs: Int = _ + private var maxAllowRunningJobs: Int = 0 private var maxAskExecutorTimes: Long = 0L def setMaxRunningJobs(maxRunningJobs: Int): Unit = this.maxRunningJobs = maxRunningJobs def getMaxRunningJobs: Int = maxRunningJobs + def setMaxAllowRunningJobs(maxAllowRunningJobs: Int): Unit = this.maxAllowRunningJobs = + maxAllowRunningJobs + + def getMaxAllowRunningJobs: Int = + if (maxAllowRunningJobs <= 0) maxRunningJobs else Math.min(maxAllowRunningJobs, maxRunningJobs) + def setMaxAskExecutorTimes(maxAskExecutorTimes: Long): Unit = this.maxAskExecutorTimes = maxAskExecutorTimes diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala index 50dce2ca12..165a274362 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala @@ -41,7 +41,6 @@ abstract class Consumer(schedulerContext: SchedulerContext, executeService: Exec def start(): Unit def shutdown(): Unit = { - logger.info(s"$toString is ready to stop!") terminate = true logger.info(s"$toString stopped!") } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala index f3471b07dd..be1716f238 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala @@ -19,6 +19,11 @@ package org.apache.linkis.scheduler.queue abstract class GroupFactory { + /** + * Create a Group and set the concurrency limit of the group + * @param event + * @return + */ def getOrCreateGroup(event: SchedulerEvent): Group def getGroup(groupName: String): Group diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala index b0bbfd3c2b..8bea7e52b1 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala @@ -40,7 +40,12 @@ class LoopArrayQueue(var group: Group) extends ConsumeQueue with Logging { override def getWaitingEvents: Array[SchedulerEvent] = { eventQueue synchronized { - toIndexedSeq.filter(x => x.getState.equals(SchedulerEventState.Inited)).toArray + toIndexedSeq + .filter(x => + x.getState.equals(SchedulerEventState.Inited) || x.getState + .equals(SchedulerEventState.Scheduled) + ) + .toArray } } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala index 4edc1d5d17..26087d99f0 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala @@ -38,4 +38,8 @@ object SchedulerEventState extends Enumeration { SchedulerEventState.withName(jobState) ) + def isInitedByStr(jobState: String): Boolean = SchedulerEventState.withName(jobState) == Inited + + def isRunningByStr(jobState: String): Boolean = isRunning(SchedulerEventState.withName(jobState)) + } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala index 2a40c2517b..fcab44a731 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala @@ -27,6 +27,7 @@ import org.apache.linkis.scheduler.executer.Executor import org.apache.linkis.scheduler.future.{BDPFuture, BDPFutureTask} import org.apache.linkis.scheduler.queue._ +import java.util import java.util.concurrent.{ExecutorService, Future} import scala.beans.BeanProperty @@ -73,6 +74,8 @@ class FIFOUserConsumer( override def getRunningEvents: Array[SchedulerEvent] = getEvents(e => e.isRunning || e.isWaitForRetry) + protected def getSchedulerContext: SchedulerContext = schedulerContext + private def getEvents(op: SchedulerEvent => Boolean): Array[SchedulerEvent] = { val result = ArrayBuffer[SchedulerEvent]() runningJobs.filter(_ != null).filter(x => op(x)).foreach(result += _) @@ -82,16 +85,28 @@ class FIFOUserConsumer( override def run(): Unit = { Thread.currentThread().setName(s"${toString}Thread") logger.info(s"$toString thread started!") - while (!terminate) { - Utils.tryAndError(loop()) - Utils.tryAndError(Thread.sleep(10)) + while (!terminate) Utils.tryAndError { + loop() + Thread.sleep(10) } logger.info(s"$toString thread stopped!") } protected def askExecutorGap(): Unit = {} + /** + * Task scheduling interception is used to judge the rules of task operation, and to judge other + * task rules based on Group. For example, Entrance makes Creator-level task judgment. + */ + protected def runScheduleIntercept(): Boolean = { + true + } + protected def loop(): Unit = { + if (!runScheduleIntercept()) { + Utils.tryQuietly(Thread.sleep(1000)) + return + } var isRetryJob = false def getWaitForRetryEvent: Option[SchedulerEvent] = { val waitForRetryJobs = runningJobs.filter(job => job != null && job.isJobCanRetry) @@ -108,8 +123,9 @@ class FIFOUserConsumer( } var event: Option[SchedulerEvent] = getWaitForRetryEvent if (event.isEmpty) { - val completedNums = runningJobs.filter(job => job == null || job.isCompleted) - if (completedNums.length < 1) { + val maxAllowRunningJobs = fifoGroup.getMaxAllowRunningJobs + val currentRunningJobs = runningJobs.count(e => e != null && !e.isCompleted) + if (maxAllowRunningJobs <= currentRunningJobs) { Utils.tryQuietly(Thread.sleep(1000)) // TODO 还可以优化，通过实现JobListener进行优化 return } @@ -119,7 +135,12 @@ class FIFOUserConsumer( if ( takeEvent.exists(e => Utils.tryCatch(e.turnToScheduled()) { t => - takeEvent.get.asInstanceOf[Job].onFailure("Job状态翻转为Scheduled失败！", t) + takeEvent.get + .asInstanceOf[Job] + .onFailure( + "Failed to change the job status to Scheduled(Job状态翻转为Scheduled失败)", + t + ) false } ) @@ -174,7 +195,7 @@ class FIFOUserConsumer( ) ) case error: Throwable => - job.onFailure("请求引擎失败，可能是由于后台进程错误!请联系管理员", error) + job.onFailure("Failed to request EngineConn", error) if (job.isWaitForRetry) { logger.warn(s"Ask executor for Job $job failed, wait for the next retry!", error) if (!isRetryJob) putToRunningJobs(job) @@ -188,8 +209,35 @@ class FIFOUserConsumer( runningJobs(index) = job } + protected def scanAllRetryJobsAndRemove(): util.List[Job] = { + val jobs = new util.ArrayList[Job]() + for (index <- runningJobs.indices) { + val job = runningJobs(index) + if (job != null && job.isJobCanRetry) { + jobs.add(job) + runningJobs(index) = null + logger.info(s"Job $job can retry, remove from runningJobs") + } + } + jobs + } + override def shutdown(): Unit = { future.cancel(true) + val waitEvents = queue.getWaitingEvents + if (waitEvents.nonEmpty) { + waitEvents.foreach { + case job: Job => + job.onFailure("Your job will be marked as canceled because the consumer be killed", null) + case _ => + } + } + + this.runningJobs.foreach { job => + if (job != null && !job.isCompleted) { + job.onFailure("Your job will be marked as canceled because the consumer be killed", null) + } + } super.shutdown() } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java index 66fafeacda..74950c15fe 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java @@ -39,7 +39,7 @@ public class LinkisStorageConf { public static final String FILE_TYPE = CommonVars.apply( "wds.linkis.storage.file.type", - "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql") + "dolphin,sql,scala,py,hql,python,out,log,text,txt,sh,jdbc,ngql,psql,fql,tsql") .getValue(); private static volatile String[] fileTypeArr = null; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java index ba4d877f3a..d98be40337 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java @@ -91,14 +91,17 @@ public void addMetaData(MetaData metaData) throws IOException { private String compact(String[] row) { String quotationMarks = "\""; + String dealNewlineSymbolMarks = "\n"; StringBuilder rowBuilder = new StringBuilder(); for (String value : row) { - String decoratedValue = - StringUtils.isBlank(value) - ? value - : quoteRetouchEnable - ? quotationMarks + value.replaceAll(quotationMarks, "") + quotationMarks - : value; + String decoratedValue = value; + if (StringUtils.isNotBlank(value)) { + if (quoteRetouchEnable) { + decoratedValue = quotationMarks + value.replaceAll(quotationMarks, "") + quotationMarks; + } + decoratedValue = decoratedValue.replaceAll(dealNewlineSymbolMarks, " "); + logger.debug("decorateValue with input: {} output: {} ", value, decoratedValue); + } rowBuilder.append(decoratedValue).append(delimiter); } if (rowBuilder.length() > 0 && rowBuilder.toString().endsWith(delimiter)) { diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java index ad9e0ee882..6808f693ec 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java @@ -26,6 +26,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.linkis.storage.domain.Dolphin.LINKIS_NULL; + public enum DataType { NullType("void", 0), StringType("string", 12), @@ -63,7 +65,8 @@ public enum DataType { public static final String LOWCASE_NULL_VALUE = "null"; // TODO Change to fine-grained regular expressions(改为精细化正则表达式) - public static final Pattern DECIMAL_REGEX = Pattern.compile("^decimal\\(\\d*\\,\\d*\\)"); + public static final Pattern DECIMAL_REGEX = + Pattern.compile("^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)"); public static final Pattern SHORT_REGEX = Pattern.compile("^short.*"); public static final Pattern INT_REGEX = Pattern.compile("^int.*"); @@ -130,7 +133,11 @@ public static DataType toDataType(String dataType) { } public static Object toValue(DataType dataType, String value) { + Object result = null; + if (isLinkisNull(value)) { + return result; + } try { switch (dataType) { case NullType: @@ -187,12 +194,16 @@ public static Object toValue(DataType dataType, String value) { result = value; } } catch (Exception e) { - logger.debug("Failed to " + value + " switch to dataType:", e); + logger.debug("Failed to {} switch to dataType:", value, e); result = value; } return result; } + public static boolean isLinkisNull(String value) { + return value == null || value.equals(LINKIS_NULL); + } + public static boolean isNull(String value) { return value == null || value.equals(NULL_VALUE) || value.trim().equals(""); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java index b6badd2849..35c71295e4 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java @@ -59,6 +59,9 @@ public class Dolphin { public static final String NULL = "NULL"; public static final byte[] NULL_BYTES = "NULL".getBytes(Charset.forName("utf-8")); + public static final String LINKIS_NULL = "LINKIS_NULL"; + public static final byte[] LINKIS_NULL_BYTES = LINKIS_NULL.getBytes(Charset.forName("utf-8")); + public static final int INT_LEN = 10; public static final int FILE_EMPTY = 31; @@ -79,6 +82,14 @@ public static String getString(byte[] bytes, int start, int len) { return new String(bytes, start, len, Dolphin.CHAR_SET); } + public static String toStringValue(String value) { + if (LINKIS_NULL.equals(value)) { + return NULL; + } else { + return value; + } + } + /** * Read an integer value that converts the array to a byte of length 10 bytes * 读取整数值，该值为将数组转换为10字节长度的byte diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java index 3c82ceb523..fad0d83a12 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java @@ -20,7 +20,10 @@ public enum StorageErrorCode { /** */ - FS_NOT_INIT(53001, "please init first(请先初始化)"); + FS_NOT_INIT(53001, "please init first"), + + INCONSISTENT_DATA(53001, "Inconsistent row data read,read %s,need rowLen %s"), + FS_OOM(53002, "OOM occurred while reading the file"); StorageErrorCode(int errorCode, String message) { this.code = errorCode; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java index 749ec9a24e..3047b715a0 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java @@ -99,6 +99,7 @@ public static ResultSetReader getTableResultReader(String res) { } Fs fs = FSFactory.getFs(resPath); + logger.info("Try to init Fs with path:{}", resPath.getPath()); try { fs.init(null); InputStream read = fs.read(resPath); diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java index 35f7483c82..c0222cc848 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java @@ -33,7 +33,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +52,6 @@ public class StorageResultSetReader private Fs fs; private final int READ_CACHE = 1024; - private final byte[] bytes = new byte[READ_CACHE]; public StorageResultSetReader(ResultSet resultSet, InputStream inputStream) { super(resultSet, inputStream); @@ -84,21 +82,18 @@ public byte[] readLine() { return null; } - byte[] rowBuffer = new byte[0]; int len = 0; - - while (rowLen > 0 && len >= 0) { - if (rowLen > READ_CACHE) { - len = StorageUtils.readBytes(inputStream, bytes, READ_CACHE); - } else { - len = StorageUtils.readBytes(inputStream, bytes, rowLen); - } - - if (len > 0) { - rowLen -= len; - rowBuffer = Arrays.copyOf(rowBuffer, rowBuffer.length + len); - System.arraycopy(bytes, 0, rowBuffer, rowBuffer.length - len, len); - } + byte[] rowBuffer = null; + try { + rowBuffer = new byte[rowLen]; + len = StorageUtils.readBytes(inputStream, rowBuffer, rowLen); + } catch (OutOfMemoryError error) { + logger.error("Result set read oom, read size {} Byte", rowLen); + throw new RuntimeException(error); + } + if (len != rowLen) { + throw new RuntimeException( + "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"); } rowCount++; return rowBuffer; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java index f13d42b0b6..c8270714ba 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java @@ -19,9 +19,6 @@ import org.apache.linkis.common.io.Record; import org.apache.linkis.storage.resultset.ResultRecord; -import org.apache.linkis.storage.utils.StorageUtils; - -import java.util.Arrays; public class TableRecord implements ResultRecord { @@ -35,10 +32,4 @@ public TableRecord(Object[] row) { public Record cloneRecord() { return new TableRecord(row.clone()); } - - public String[] tableRecordToString(String nullValue) { - return Arrays.stream(row) - .map(col -> StorageUtils.colToString(col, nullValue)) - .toArray(String[]::new); - } } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java index 7b7838fd25..7e1d6c35fe 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java @@ -52,13 +52,13 @@ public TableMetaData createMetaData(byte[] bytes) { List columns = new ArrayList<>(); for (int i = 0; i < colArray.length; i += 3) { int len = Integer.parseInt(colArray[i]); - String colName = Dolphin.getString(bytes, index, len); + String colName = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)); index += len; len = Integer.parseInt(colArray[i + 1]); - String colType = Dolphin.getString(bytes, index, len); + String colType = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)); index += len; len = Integer.parseInt(colArray[i + 2]); - String colComment = Dolphin.getString(bytes, index, len); + String colComment = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)); index += len; columns.add(new Column(colName, DataType.toDataType(colType), colComment)); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java index 6abe4c56d5..5f40aa33f3 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java @@ -61,7 +61,7 @@ private byte[] lineToBytes(Object[] line) { int colByteLen = 0; int length = 0; for (Object data : line) { - byte[] bytes = data == null ? Dolphin.NULL_BYTES : Dolphin.getBytes(data); + byte[] bytes = data == null ? Dolphin.LINKIS_NULL_BYTES : Dolphin.getBytes(data); dataBytes.add(bytes); byte[] colBytes = Dolphin.getBytes(bytes.length); colIndex.add(colBytes); diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java index cee72dfcd7..0ed650186d 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java @@ -135,6 +135,7 @@ static FileSource create(FsPath fsPath, InputStream is) { } static FileSplit createResultSetFileSplit(FsPath fsPath, InputStream is) { + logger.info("try create result set file split with path:{}", fsPath.getPath()); ResultSet resultset = ResultSetFactory.getInstance().getResultSetByPath(fsPath); ResultSetReader resultsetReader = ResultSetReaderFactory.getResultSetReader(resultset, is); return new FileSplit(resultsetReader, resultset.resultSetType()); diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java index d8562b7c5b..fb064a8f4f 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java @@ -17,6 +17,7 @@ package org.apache.linkis.storage.source; +import org.apache.linkis.storage.domain.Dolphin; import org.apache.linkis.storage.resultset.table.TableRecord; import org.apache.linkis.storage.utils.StorageUtils; @@ -36,9 +37,18 @@ record -> { .map( r -> { if (r == null || r.equals("NULL")) { - return nullValue; + if (nullValue.equals(Dolphin.LINKIS_NULL)) { + return r; + } else { + return nullValue; + } } else if (r.equals("")) { - return getParams().getOrDefault("nullValue", ""); + String emptyValue = getParams().getOrDefault("nullValue", ""); + if (emptyValue.equals(Dolphin.LINKIS_NULL)) { + return ""; + } else { + return nullValue; + } } else if (r instanceof Double) { return StorageUtils.doubleToString((Double) r); } else { diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java index 0f93cdb6ab..2809c83eec 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java @@ -96,6 +96,30 @@ public static void createNewFileWithFileSystem( } } + /** + * create new file and set file owner by FileSystem + * + * @param fileSystem + * @param filePath + * @param user + * @param createParentWhenNotExists + */ + public static void createNewFileAndSetOwnerWithFileSystem( + FileSystem fileSystem, FsPath filePath, String user, boolean createParentWhenNotExists) + throws Exception { + if (!fileSystem.exists(filePath)) { + if (!fileSystem.exists(filePath.getParent())) { + if (!createParentWhenNotExists) { + throw new IOException( + "parent dir " + filePath.getParent().getPath() + " dose not exists."); + } + mkdirs(fileSystem, filePath.getParent(), user); + } + fileSystem.createNewFile(filePath); + fileSystem.setOwner(filePath, user); + } + } + /** * Recursively create a directory * @@ -133,4 +157,39 @@ public static boolean mkdirs(FileSystem fileSystem, FsPath dest, String user) th } return true; } + + /** + * Recursively create a directory(递归创建目录) add owner info + * + * @param fileSystem + * @param dest + * @param user + * @throws IOException + * @return + */ + public static boolean mkdirsAndSetOwner(FileSystem fileSystem, FsPath dest, String user) + throws IOException { + FsPath parentPath = dest.getParent(); + Stack dirsToMake = new Stack<>(); + dirsToMake.push(dest); + while (!fileSystem.exists(parentPath)) { + dirsToMake.push(parentPath); + + if (Objects.isNull(parentPath.getParent())) { + // parent path of root is null + break; + } + + parentPath = parentPath.getParent(); + } + if (!fileSystem.canExecute(parentPath)) { + throw new IOException("You have not permission to access path " + dest.getPath()); + } + while (!dirsToMake.empty()) { + FsPath path = dirsToMake.pop(); + fileSystem.mkdir(path); + fileSystem.setOwner(path, user); + } + return true; + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/pom.xml index 9d0cbbb849..76723c4adb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/pom.xml +++ b/linkis-computation-governance/linkis-client/linkis-cli/pom.xml @@ -62,6 +62,12 @@ ${project.artifactId}-${project.version} + + + true + ${basedir}/src/main/resources + + org.apache.maven.plugins diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java index 592725f857..07df8b79ad 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java @@ -46,9 +46,11 @@ import org.apache.commons.lang3.StringUtils; +import java.io.*; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -156,6 +158,23 @@ public static CliCtx buildCtx(String[] args) throws LinkisClientRuntimeException .init(); logger.info("==========std_var============\n" + CliUtils.GSON.toJson(varAccess)); - return new CliCtxImpl(params.getCmdType(), template, varAccess, new HashMap<>()); + Properties props = new Properties(); + try (InputStream inputStream = + CtxBuilder.class.getClassLoader().getResourceAsStream("version.properties")) { + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + try (BufferedReader bufferedReader = new BufferedReader(reader)) { + props.load(bufferedReader); + } + } + } catch (Exception e) { + logger.warn("Failed to load version info", e); + } + + String verion = props.getProperty(CliKeys.VERSION); + + Map extraMap = new HashMap<>(); + extraMap.put(CliKeys.VERSION, verion); + + return new CliCtxImpl(params.getCmdType(), template, varAccess, extraMap); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java index f13c7399fc..1fb21043a1 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java @@ -26,9 +26,11 @@ import org.apache.linkis.cli.application.exception.CommandException; import org.apache.linkis.cli.application.interactor.command.CmdTemplateFactory; import org.apache.linkis.cli.application.interactor.command.template.UniversalCmdTemplate; +import org.apache.linkis.cli.application.interactor.job.help.HelpJob; import org.apache.linkis.cli.application.interactor.job.interactive.InteractiveJob; import org.apache.linkis.cli.application.interactor.job.jobcmd.JobCmdJob; import org.apache.linkis.cli.application.interactor.job.once.LinkisOnceJob; +import org.apache.linkis.cli.application.interactor.job.version.VersionJob; import org.apache.linkis.cli.application.operator.OperManager; import org.apache.linkis.cli.application.operator.once.OnceOperBuilder; import org.apache.linkis.cli.application.operator.ujes.LinkisOperBuilder; @@ -49,6 +51,8 @@ public class LinkisClientApplication { private static Logger logger = LoggerFactory.getLogger(LinkisClientApplication.class); + private static boolean showHelp = false; + public static void main(String[] args) { /* generate template @@ -64,7 +68,9 @@ public static void main(String[] args) { } catch (CommandException e) { CmdTemplate template = CmdTemplateFactory.getTemplateOri(e.getCmdType()); if (template != null) { - printHelp(template); + HelpInfoModel model = new HelpInfoModel(); + model.buildModel(ctx.getTemplate()); + new HelpPresenter().present(model); } LoggerManager.getInformationLogger().error("Failed to build CliCtx", e); System.exit(-1); @@ -80,7 +86,11 @@ public static void main(String[] args) { run job */ Job job; - if (isJobCmd(ctx)) { + if (isVersionCmd(ctx)) { + job = new VersionJob(); + } else if (isHelp(ctx)) { + job = new HelpJob(); + } else if (isJobCmd(ctx)) { job = new JobCmdJob(); } else if (isOnceCmd(ctx)) { job = new LinkisOnceJob(); @@ -134,21 +144,34 @@ public Map getExtraMessage() { } } - private static void printHelp(CmdTemplate template) { - HelpInfoModel model = new HelpInfoModel(); - model.buildModel(template); - new HelpPresenter().present(model); - } - private static void printIndicator(JobResult jobResult) { if (jobResult.isSuccess()) { LoggerManager.getPlaintTextLogger().info(CliConstants.SUCCESS_INDICATOR); } else { + LoggerManager.getPlaintTextLogger().info(jobResult.getMessage()); + StringBuilder b = new StringBuilder(); + for (Map.Entry e : jobResult.getExtraMessage().entrySet()) { + b.append(e.getKey()).append(":").append(e.getValue()).append(System.lineSeparator()); + } + LoggerManager.getPlaintTextLogger().info(b.toString()); LoggerManager.getPlaintTextLogger().info(CliConstants.FAILURE_INDICATOR); - LoggerManager.getPlaintTextLogger().error(jobResult.getMessage()); } } + private static Boolean isHelp(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_HELP_OPT)) { + return true; + } + return false; + } + + private static Boolean isVersionCmd(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.VERSION)) { + return true; + } + return false; + } + private static Boolean isJobCmd(CliCtx ctx) { if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_KILL_OPT) || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_STATUS_OPT) diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java index 5c04cbcd64..966836e0bf 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java @@ -38,6 +38,8 @@ public class CliKeys { public static final String DEFAULT_CONFIG_FILE_NAME_KEY = "conf.file"; public static final String LINUX_USER_KEY = "user.name"; + public static final String VERSION = "cli.version"; + /** Configurable */ /* execution type diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java index 8483003132..170f1a8f86 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java @@ -47,4 +47,5 @@ public class LinkisKeys { public static final String KEY_YARN_QUEUE = "wds.linkis.rm.yarnqueue"; public static final String KEY_HIVE_RESULT_DISPLAY_TBALE = "hive.resultset.use.unique.column.names"; + public static final String CLI_VERSION = "cli.version"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java index f761a6a47a..83b42032cf 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java @@ -271,6 +271,15 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab "specify jobContent map. Input format: -jobContentMap key1=value1 -jobContentMap key2=value2", true); + protected Flag versionFlag = + flag( + CliKeys.VERSION, + CliKeys.VERSION, + new String[] {"--version"}, + "show version", + true, + false); + protected Parameter argumentsParas = parameter( CliKeys.LINKIS_CLIENT_COMMON, @@ -286,6 +295,9 @@ public UniversalCmdTemplate() { @Override public void checkParams() throws CommandException { + if (versionFlag.hasVal()) { + return; + } int cnt = 0; if (statusOpt.hasVal()) { cnt++; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java new file mode 100644 index 0000000000..943c54ed63 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.help; + +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.present.HelpPresenter; +import org.apache.linkis.cli.application.present.model.HelpInfoModel; + +import java.util.HashMap; +import java.util.Map; + +public class HelpJob implements Job { + private CliCtx ctx; + + @Override + public void build(CliCtx ctx) { + this.ctx = ctx; + } + + @Override + public JobResult run() { + HelpInfoModel model = new HelpInfoModel(); + model.buildModel(ctx.getTemplate()); + new HelpPresenter().present(model); + return new JobResult() { + @Override + public Boolean isSuccess() { + return true; + } + + @Override + public String getMessage() { + return ""; + } + + @Override + public Map getExtraMessage() { + return new HashMap<>(); + } + }; + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java index a097b1b25b..12e491c5af 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java @@ -88,17 +88,17 @@ public JobResult run() { LinkisOperResultAdapter jobInfoResult = oper.queryJobInfo(submitResult.getUser(), submitResult.getJobID()); oper.queryJobStatus( - jobInfoResult.getUser(), jobInfoResult.getJobID(), jobInfoResult.getStrongerExecId()); + submitResult.getUser(), submitResult.getJobID(), submitResult.getStrongerExecId()); infoBuilder.setLength(0); infoBuilder .append("JobId:") - .append(jobInfoResult.getJobID()) + .append(submitResult.getJobID()) .append(System.lineSeparator()) .append("TaskId:") - .append(jobInfoResult.getJobID()) + .append(submitResult.getJobID()) .append(System.lineSeparator()) .append("ExecId:") - .append(jobInfoResult.getStrongerExecId()); + .append(submitResult.getStrongerExecId()); LoggerManager.getPlaintTextLogger().info(infoBuilder.toString()); infoBuilder.setLength(0); @@ -137,7 +137,9 @@ public JobResult run() { logRetriever.retrieveLogAsync(); // wait complete - jobInfoResult = waitJobComplete(submitResult.getUser(), submitResult.getJobID()); + jobInfoResult = + waitJobComplete( + submitResult.getUser(), submitResult.getJobID(), submitResult.getStrongerExecId()); logRetriever.waitIncLogComplete(); // get result-set @@ -205,19 +207,19 @@ private JobResult getResult( return result; } - private LinkisOperResultAdapter waitJobComplete(String user, String jobId) + private LinkisOperResultAdapter waitJobComplete(String user, String jobId, String execId) throws LinkisClientRuntimeException { int retryCnt = 0; final int MAX_RETRY = 30; LinkisOperResultAdapter jobInfoResult = oper.queryJobInfo(user, jobId); - oper.queryJobStatus(user, jobId, jobInfoResult.getStrongerExecId()); + oper.queryJobStatus(user, jobId, execId); while (!jobInfoResult.getJobStatus().isJobFinishedState()) { // query progress try { jobInfoResult = oper.queryJobInfo(user, jobId); - oper.queryJobStatus(user, jobId, jobInfoResult.getStrongerExecId()); + oper.queryJobStatus(user, jobId, execId); } catch (Exception e) { logger.warn("", e); retryCnt++; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java index 0c8a3db539..2b0b20188a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java @@ -26,6 +26,7 @@ import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory; import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; import org.apache.commons.lang3.StringUtils; @@ -135,7 +136,12 @@ public static InteractiveJobDesc build(CliCtx ctx) { } if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { - code = CliUtils.readFile(codePath); + try { + code = CliUtils.readFile(codePath); + } catch (Exception e) { + LoggerManager.getInformationLogger().error("Failed to read file", e); + throw e; + } } executionMap.put(LinkisKeys.KEY_CODE, code); @@ -143,6 +149,9 @@ public static InteractiveJobDesc build(CliCtx ctx) { labelMap.put(LinkisKeys.KEY_CODETYPE, runType); labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); + if (ctx.getExtraMap().containsKey(CliKeys.VERSION)) { + sourceMap.put(LinkisKeys.CLI_VERSION, ctx.getExtraMap().get(CliKeys.VERSION)); + } runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); desc.setCreator(creator); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java new file mode 100644 index 0000000000..599f97904f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.version; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import java.util.HashMap; +import java.util.Map; + +public class VersionJob implements Job { + private CliCtx ctx; + + @Override + public void build(CliCtx cliCtx) { + this.ctx = cliCtx; + } + + @Override + public JobResult run() { + String version = (String) ctx.getExtraMap().get(CliKeys.VERSION); + Map extraMap = new HashMap<>(); + extraMap.put(CliKeys.VERSION, version); + LoggerManager.getPlaintTextLogger().info("Version=" + version); + return new VersionJobResult(true, "ok", extraMap); + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java similarity index 50% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java index 103a544772..e2f12cd7c2 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java @@ -15,28 +15,35 @@ * limitations under the License. */ -package org.apache.linkis.ujes.jdbc.hook.impl +package org.apache.linkis.cli.application.interactor.job.version; -import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain -import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook +import org.apache.linkis.cli.application.entity.job.JobResult; -import java.util.Locale +import java.util.Map; -class NoLimitExecutionHook extends JDBCDriverPreExecutionHook { +public class VersionJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMsg; - override def callPreExecutionHook(sql: String, skip: Boolean): String = { - if (UJESSQLDriverMain.LIMIT_ENABLED.equalsIgnoreCase("false")) { - var noLimitSql = "--set ide.engine.no.limit.allow=true\n" + sql - val lowerCaseLimitSql = noLimitSql.toLowerCase() - if (lowerCaseLimitSql.contains("limit ") && lowerCaseLimitSql.contains("tableausql")) { - val lastIndexOfLimit = lowerCaseLimitSql.lastIndexOf("limit ") - noLimitSql = noLimitSql.substring(0, lastIndexOfLimit) - } - noLimitSql - } else { - sql - } + public VersionJobResult(Boolean success, String message, Map extraMsg) { + this.success = success; + this.message = message; + this.extraMsg = extraMsg; + } + + @Override + public Boolean isSuccess() { + return success; + } + @Override + public String getMessage() { + return message; } + @Override + public Map getExtraMessage() { + return extraMsg; + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java index c4f7a3aadc..bfebf62c71 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java @@ -127,7 +127,7 @@ public LinkisOperResultAdapter submit(InteractiveJobDesc jobDesc) // jobExecuteResult = client.execute(jobExecuteAction); jobSubmitResult = client.submit(jobSubmitAction); - logger.info("Response info from Linkis: \n{}", CliUtils.GSON.toJson(jobSubmitAction)); + logger.info("Response info from Linkis: \n{}", CliUtils.GSON.toJson(jobSubmitResult)); } catch (Exception e) { // must throw if exception @@ -269,7 +269,7 @@ private JobInfoResult queryJobInfoInternal(String user, String taskID) } String msg = MessageFormat.format( - "Get job info failed. retry time : {0}/{1}. taskID={0}, Reason: {1}", + "Get job info failed. retry time : {0}/{1}. taskID={2}, Reason: {3}", retryTime, MAX_RETRY_TIME, taskID, reason); logger.debug( @@ -299,7 +299,7 @@ private JobInfoResult queryJobInfoInternal(String user, String taskID) if (jobInfoResult == null) { reason = "JobInfoResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; reason += jobInfoResult.getMessage(); } String msg = diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java index 6c746ff57c..63fb004db5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java @@ -162,6 +162,10 @@ public String getStrongerExecId() { return null; } String execId = null; + + if (result instanceof JobSubmitResult) { + execId = ((JobSubmitResult) result).getExecID(); + } if (result instanceof JobInfoResult) { if (result != null && ((JobInfoResult) result).getTask() != null diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java index 9a54699165..c2d47e2b7a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java @@ -32,7 +32,6 @@ public static void writeToFile( String pathName, String fileName, String content, Boolean overWrite) { File dir = new File(pathName); - File file = new File(fileName); if (!dir.exists()) { try { @@ -47,6 +46,8 @@ public static void writeToFile( } } + File file = new File(dir.getAbsolutePath() + File.separator + fileName); + if (overWrite || !file.exists()) { try { file.createNewFile(); diff --git a/linkis-public-enhancements/linkis-error-code/linkis-error-code-server/src/test/resources/linkis.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties similarity index 86% rename from linkis-public-enhancements/linkis-error-code/linkis-error-code-server/src/test/resources/linkis.properties rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties index 1c575edc5b..0da37e5c03 100644 --- a/linkis-public-enhancements/linkis-error-code/linkis-error-code-server/src/test/resources/linkis.properties +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties @@ -1,4 +1,4 @@ -# +# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -13,9 +13,4 @@ # limitations under the License. # -#wds.linkis.test.mode=true -wds.linkis.server.version=v1 - -#test -wds.linkis.test.mode=true -wds.linkis.test.user=hadoop \ No newline at end of file +cli.version=${project.version} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala index 6b41b4c62b..9eb748691e 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/ResultSetAction.scala @@ -35,6 +35,9 @@ object ResultSetAction { private var pageSize: Int = _ private var charset: String = Configuration.BDP_ENCODING.getValue + // default value is :org.apache.linkis.storage.domain.Dolphin.LINKIS_NULL + private var nullValue: String = "LINKIS_NULL" + def setUser(user: String): Builder = { this.user = user this @@ -60,6 +63,11 @@ object ResultSetAction { this } + def setNullValue(nullValue: String): Builder = { + this.nullValue = nullValue + this + } + def build(): ResultSetAction = { if (user == null) throw new UJESClientBuilderException("user is needed!") if (path == null) throw new UJESClientBuilderException("path is needed!") @@ -68,6 +76,7 @@ object ResultSetAction { if (page > 0) resultSetAction.setParameter("page", page) if (pageSize > 0) resultSetAction.setParameter("pageSize", pageSize) resultSetAction.setParameter("charset", charset) + resultSetAction.setParameter("nullValue", nullValue) resultSetAction.setUser(user) resultSetAction } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/TemplateConfKey.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/TemplateConfKey.java new file mode 100644 index 0000000000..13cbac5577 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/TemplateConfKey.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.entity; + +public class TemplateConfKey { + + private String templateUuid; + + private String key; + + private String templateName; + + private String configValue; + + public String getTemplateUuid() { + return templateUuid; + } + + public void setTemplateUuid(String templateUuid) { + this.templateUuid = templateUuid; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getTemplateName() { + return templateName; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + public String getConfigValue() { + return configValue; + } + + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + @Override + public String toString() { + return "TemplateKey{" + + "templateUuid='" + + templateUuid + + '\'' + + ", key='" + + key + + '\'' + + ", templateName='" + + templateName + + '\'' + + ", configValue='" + + configValue + + '\'' + + '}'; + } +} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java index d5d97aa364..46fa8a69ef 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java @@ -49,6 +49,9 @@ public class JobRequest { /** result location */ private String resultLocation; + /** Task status updates is ordered, if false, not checked */ + private Boolean updateOrderFlag = true; + private String observeInfo; private Map metrics = new HashMap<>(); @@ -205,6 +208,14 @@ public void setObserveInfo(String observeInfo) { this.observeInfo = observeInfo; } + public Boolean getUpdateOrderFlag() { + return updateOrderFlag; + } + + public void setUpdateOrderFlag(Boolean updateOrderFlag) { + this.updateOrderFlag = updateOrderFlag; + } + @Override public String toString() { return "JobRequest{" diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfRequest.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfRequest.java new file mode 100644 index 0000000000..e8b566cda1 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfRequest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.protocol.conf; + +import org.apache.linkis.protocol.message.RequestProtocol; + +public class TemplateConfRequest implements RequestProtocol { + + private String templateUuid; + + private String templateName; + + public TemplateConfRequest(String templateUuid, String templateName) { + this.templateUuid = templateUuid; + this.templateName = templateName; + } + + public TemplateConfRequest(String templateUuid) { + this.templateUuid = templateUuid; + } + + public String getTemplateUuid() { + return templateUuid; + } + + public void setTemplateUuid(String templateUuid) { + this.templateUuid = templateUuid; + } + + public String getTemplateName() { + return templateName; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } +} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfResponse.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfResponse.java new file mode 100644 index 0000000000..8822fe988d --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/protocol/conf/TemplateConfResponse.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.protocol.conf; + +import org.apache.linkis.governance.common.entity.TemplateConfKey; + +import java.util.ArrayList; +import java.util.List; + +public class TemplateConfResponse { + + private List list = new ArrayList<>(); + + public List getList() { + return list; + } + + public void setList(List list) { + this.list = list; + } +} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala index a4671eaa17..b8b156173b 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala @@ -90,10 +90,4 @@ object GovernanceCommonConf { val EC_APP_MANAGE_MODE = CommonVars("linkis.ec.app.manage.mode", "attach") - val SCALA_PARSE_APPEND_CODE_ENABLED = - CommonVars("linkis.scala.parse.append.code.enable", true).getValue - - val SCALA_PARSE_APPEND_CODE = - CommonVars("linkis.scala.parse.append.code", "val linkisVar=1").getValue - } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala index 544dfcdab6..ec7bb9e80a 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala @@ -17,7 +17,7 @@ package org.apache.linkis.governance.common.exception -import org.apache.linkis.common.exception.{ErrorException, ExceptionLevel, LinkisRuntimeException} +import org.apache.linkis.common.exception.{ExceptionLevel, LinkisRuntimeException} class GovernanceErrorException(errorCode: Int, errorMsg: String) extends LinkisRuntimeException(errorCode, errorMsg) { diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala index 87576d5e48..64ece62fd7 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala @@ -86,6 +86,11 @@ abstract class CombinedEngineCodeParser extends CodeParser { } +/** + * Scala is no longer using Parser but instead using EmptyParser. If there is a comment at the end, + * it will cause the task to become stuck + */ +@deprecated class ScalaCodeParser extends SingleCodeParser with Logging { override val codeType: CodeType = CodeType.Scala @@ -109,11 +114,9 @@ class ScalaCodeParser extends SingleCodeParser with Logging { case _ => } if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n")) - - // Append code `val linkisVar=1` in ends to prevent bugs that do not exit tasks for a long time - if (GovernanceCommonConf.SCALA_PARSE_APPEND_CODE_ENABLED) { - codeBuffer.append(GovernanceCommonConf.SCALA_PARSE_APPEND_CODE) - } + // Make sure the last line is not a comment + codeBuffer.append("\n") + codeBuffer.append("val linkisVar=123") codeBuffer.toArray } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala index 2e44739787..df197ddb2c 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala @@ -51,3 +51,10 @@ class RequestOneJob extends JobReq { } case class RequestAllJob(instance: String) extends JobReq + +case class RequestFailoverJob( + reqMap: util.Map[String, java.lang.Long], + statusList: util.List[String], + startTimestamp: Long, + limit: Int = 10 +) extends JobReq diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala index db7045baec..04adf3446c 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala @@ -29,7 +29,8 @@ class ScalaCodeParserTest { "val codeBuffer = new ArrayBuffer[String]()\n val statementBuffer = new ArrayBuffer[String]()" val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(scalaCode) - Assertions.assertTrue(array.length == 2) + Assertions.assertTrue(array.size == 3) + } @Test @@ -40,7 +41,7 @@ class ScalaCodeParserTest { " def addInt( a:Int, b:Int )\n var sum:Int = 0\n sum = a + b\n return sum\n }" val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(abnormalCode) - Assertions.assertTrue(array.length == 2) + Assertions.assertTrue(array.length == 3) } @@ -53,7 +54,7 @@ class ScalaCodeParserTest { val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(importCode) - Assertions.assertTrue(array.length == 3) + Assertions.assertTrue(array.length == 4) } @@ -67,7 +68,7 @@ class ScalaCodeParserTest { val scalaCodeParser = new ScalaCodeParser val arrayResult1 = scalaCodeParser.parse(specialCodeExp1) - Assertions.assertTrue(arrayResult1.length == 3) + Assertions.assertTrue(arrayResult1.length == 4) val specialCodeExp2 = " @BeanProperty\n var id: Long = _\n @BeanProperty\n var status: Int = 0\n " + @@ -78,7 +79,7 @@ class ScalaCodeParserTest { ".append(data, that.data)\n .isEquals\n }" val arrayResult2 = scalaCodeParser.parse(specialCodeExp2) - Assertions.assertTrue(arrayResult2.length == 2) + Assertions.assertTrue(arrayResult2.length == 3) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala index 940973be61..ea80b625bd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala @@ -271,6 +271,7 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) TaskResponseErrorEvent(engineConnTask.getTaskId, errorExecuteResponse.message) ) transformTaskStatus(engineConnTask, ExecutionNodeStatus.Failed) + case _ => logger.warn(s"task get response is $executeResponse") } executeResponse } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index bc738d5498..ca1887e746 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -187,7 +187,7 @@ class TaskExecutionServiceImpl if (!lockService.isLockExist(requestTask.getLock)) { logger.error(s"Lock ${requestTask.getLock} not exist, cannot execute.") return ErrorExecuteResponse( - "Lock not exixt", + "Lock not exist", new EngineConnExecutorErrorException( EngineConnExecutorErrorCode.INVALID_LOCK, "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)." diff --git a/linkis-computation-governance/linkis-entrance/pom.xml b/linkis-computation-governance/linkis-entrance/pom.xml index dea4d1d4d7..bda458c356 100644 --- a/linkis-computation-governance/linkis-entrance/pom.xml +++ b/linkis-computation-governance/linkis-entrance/pom.xml @@ -90,6 +90,12 @@ ${project.version} + + org.apache.linkis + linkis-ps-common-lock + ${project.version} + + diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index 0bf27a68b3..cf520c3823 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -42,6 +42,7 @@ import org.apache.linkis.entrance.persistence.QueryPersistenceManager; import org.apache.linkis.entrance.persistence.ResultSetEngine; import org.apache.linkis.entrance.scheduler.EntranceGroupFactory; +import org.apache.linkis.entrance.scheduler.EntranceParallelConsumerManager; import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder$; @@ -51,7 +52,6 @@ import org.apache.linkis.scheduler.executer.ExecutorManager; import org.apache.linkis.scheduler.queue.ConsumerManager; import org.apache.linkis.scheduler.queue.GroupFactory; -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager; import org.apache.linkis.scheduler.queue.parallelqueue.ParallelScheduler; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -190,7 +190,7 @@ public GroupFactory groupFactory() { @Bean @ConditionalOnMissingBean public ConsumerManager consumerManager() { - return new ParallelConsumerManager( + return new EntranceParallelConsumerManager( ENTRANCE_SCHEDULER_MAX_PARALLELISM_USERS().getValue(), "EntranceJobScheduler"); } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java index cb37279c11..bee17b8ed4 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java @@ -26,4 +26,6 @@ private ServiceNameConsts() {} public static final String ENTRANCE_SERVER = "entranceServer"; public static final String ENTRANCE_INTERCEPTOR = "entranceInterceptors"; + + public static final String ENTRANCE_FAILOVER_SERVER = "entranceFailoverServer"; } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java index 2f045a1760..b5f90e3070 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java @@ -71,7 +71,11 @@ public enum EntranceErrorCodeSummary implements LinkisErrorCode { SHELL_BLACKLISTED_CODE(50081, "Shell code contains blacklisted code(shell中包含黑名单代码)"), JOB_HISTORY_FAILED_ID(50081, ""), - LOGPATH_NOT_NULL(20301, "The logPath cannot be empty(日志路径不能为空)"); + LOGPATH_NOT_NULL(20301, "The logPath cannot be empty(日志路径不能为空)"), + + FAILOVER_RUNNING_TO_CANCELLED( + 30001, + "Job {0} failover, status changed from Running to Cancelled (任务故障转移，状态从Running变更为Cancelled)"); /** (errorCode)错误码 */ private final int errorCode; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java index 2ab457747c..841a6a3fb0 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java @@ -18,14 +18,18 @@ package org.apache.linkis.entrance.restful; import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.instance.label.client.InstanceLabelClient; import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.constant.LabelValueConstant; import org.apache.linkis.protocol.label.InsLabelRefreshRequest; import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.SchedulerContext; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; @@ -46,6 +50,12 @@ public class EntranceLabelRestfulApi { private static final Logger logger = LoggerFactory.getLogger(EntranceLabelRestfulApi.class); + private EntranceServer entranceServer; + + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + this.entranceServer = entranceServer; + } @ApiOperation(value = "update", notes = "update route label", response = Message.class) @ApiOperationSupport(ignoreParameters = {"jsonNode"}) @@ -79,6 +89,16 @@ public Message updateRouteLabel(HttpServletRequest req) { insLabelRefreshRequest.setServiceInstance(Sender.getThisServiceInstance()); InstanceLabelClient.getInstance().refreshLabelsToInstance(insLabelRefreshRequest); logger.info("Finished to modify the routelabel of entry to offline"); + + logger.info("Prepare to update all not execution task instances to empty string"); + SchedulerContext schedulerContext = + entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); + if (schedulerContext instanceof EntranceSchedulerContext) { + ((EntranceSchedulerContext) schedulerContext).setOfflineFlag(true); + } + entranceServer.updateAllNotExecutionTaskInstances(true); + logger.info("Finished to update all not execution task instances to empty string"); + return Message.ok(); } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index c1479efd8a..cf9cd33653 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -29,13 +29,16 @@ import org.apache.linkis.governance.common.entity.job.JobRequest; import org.apache.linkis.manager.common.protocol.resource.ResourceWithStatus; import org.apache.linkis.protocol.constants.TaskConstant; +import org.apache.linkis.protocol.engine.JobInstance; import org.apache.linkis.protocol.engine.JobProgressInfo; import org.apache.linkis.protocol.utils.ZuulEntranceUtils; import org.apache.linkis.rpc.Sender; import org.apache.linkis.scheduler.listener.LogListener; import org.apache.linkis.scheduler.queue.Job; import org.apache.linkis.scheduler.queue.SchedulerEventState; +import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.server.Message; +import org.apache.linkis.server.conf.ServerConfiguration; import org.apache.linkis.server.security.SecurityFilter; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -61,6 +64,7 @@ import scala.Option; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import io.swagger.annotations.Api; @@ -198,6 +202,13 @@ private void pushLog(String log, Job job) { entranceServer.getEntranceContext().getOrCreateLogManager().onLogUpdate(job, log); } + private JobInstance parseHeaderToJobInstance(HttpServletRequest req) + throws JsonProcessingException { + String jobStr = + req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().getValue()); + return BDPJettyServerHelper.gson().fromJson(jobStr, JobInstance.class); + } + @ApiOperation(value = "status", notes = "get task stats", response = Message.class) @ApiImplicitParams({ @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = " task id"), @@ -209,28 +220,74 @@ public Message status( HttpServletRequest req, @PathVariable("id") String id, @RequestParam(value = "taskID", required = false) String taskID) { + ModuleUserUtils.getOperationUser(req, "job status"); Message message = null; - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "status realId: " + realId); - Option job = Option.apply(null); + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/status"); + return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", jobInstance.status()).data("execID", "").data("taskID", id); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return status is Inited", id); + String status = SchedulerEventState.Inited().toString(); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", status).data("execID", "").data("taskID", id); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } + + Option job = null; try { job = entranceServer.getJob(realId); } catch (Exception e) { - logger.warn("获取任务 {} 状态时出现错误", realId, e.getMessage()); + logger.warn("get {} status error", realId, e); + if (StringUtils.isEmpty(taskID)) { + message = + Message.error( + "Get job by ID error and cannot obtain the corresponding task status.(获取job时发生异常，不能获取相应的任务状态)"); + return message; + } long realTaskID = Long.parseLong(taskID); String status = JobHistoryHelper.getStatusByTaskID(realTaskID); message = Message.ok(); message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", status).data("execID", id); + message.data("status", status).data("execID", execID); return message; } - if (job.isDefined()) { + if (job != null && job.isDefined()) { if (job.get() instanceof EntranceJob) { ((EntranceJob) job.get()).updateNewestAccessByClientTimestamp(); } message = Message.ok(); message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", job.get().getState().toString()).data("execID", id); + message.data("status", job.get().getState().toString()).data("execID", execID); } else { message = Message.error( @@ -246,9 +303,56 @@ public Message status( @Override @RequestMapping(path = "/{id}/progress", method = RequestMethod.GET) public Message progress(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "job progress"); Message message = null; - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "progress realId: " + realId); + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/progress"); + return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progress"); + message + .data("progress", "1.0") + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return progress is 0", id); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progress"); + message + .data("progress", 0) + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } + Option job = null; try { job = entranceServer.getJob(realId); @@ -275,7 +379,7 @@ public Message progress(HttpServletRequest req, @PathVariable("id") String id) { message .data("progress", Math.abs(job.get().getProgress())) - .data("execID", id) + .data("execID", execID) .data("progressInfo", list); } } else { @@ -296,9 +400,60 @@ public Message progress(HttpServletRequest req, @PathVariable("id") String id) { @Override @RequestMapping(path = "/{id}/progressWithResource", method = RequestMethod.GET) public Message progressWithResource(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "job progressWithResource"); Message message = null; - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "progressWithResource realId: " + realId); + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + long realTaskID = Long.parseLong(id); + JobRequest jobRequest = JobHistoryHelper.getTaskByTaskID(realTaskID); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + Map metricsVo = new HashMap<>(); + buildYarnResource(jobRequest, metricsVo, message); + message + .data("progress", "1.0") + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return progress is 0 and resource is null", id); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + message + .data(TaskConstant.JOB_YARNRESOURCE, null) + .data("progress", 0) + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } Option job = null; try { job = entranceServer.getJob(realId); @@ -324,57 +479,12 @@ public Message progressWithResource(HttpServletRequest req, @PathVariable("id") message.setMethod("/api/entrance/" + id + "/progressWithResource"); JobRequest jobRequest = ((EntranceJob) job.get()).getJobRequest(); - Map metrics = jobRequest.getMetrics(); Map metricsVo = new HashMap<>(); - if (metrics.containsKey(TaskConstant.JOB_YARNRESOURCE)) { - HashMap resourceMap = - (HashMap) metrics.get(TaskConstant.JOB_YARNRESOURCE); - ArrayList resoureList = new ArrayList<>(12); - if (null != resourceMap && !resourceMap.isEmpty()) { - resourceMap.forEach( - (applicationId, resource) -> { - resoureList.add(new YarnResourceWithStatusVo(applicationId, resource)); - }); - metricsVo.put(TaskConstant.JOB_YARNRESOURCE, resoureList); - Optional cores = - resourceMap.values().stream() - .map(resource -> resource.getQueueCores()) - .reduce((x, y) -> x + y); - Optional memory = - resourceMap.values().stream() - .map(resource -> resource.queueMemory()) - .reduce((x, y) -> x + y); - float corePercent = 0.0f; - float memoryPercent = 0.0f; - if (cores.isPresent() && memory.isPresent()) { - corePercent = - cores.get().floatValue() - / EntranceConfiguration.YARN_QUEUE_CORES_MAX().getHotValue(); - memoryPercent = - memory.get().floatValue() - / (EntranceConfiguration.YARN_QUEUE_MEMORY_MAX().getHotValue().longValue() - * 1024 - * 1024 - * 1024); - } - String coreRGB = RGBUtils.getRGB(corePercent); - String memoryRGB = RGBUtils.getRGB(memoryPercent); - metricsVo.put(TaskConstant.JOB_CORE_PERCENT, corePercent); - metricsVo.put(TaskConstant.JOB_MEMORY_PERCENT, memoryPercent); - metricsVo.put(TaskConstant.JOB_CORE_RGB, coreRGB); - metricsVo.put(TaskConstant.JOB_MEMORY_RGB, memoryRGB); - - message.data(TaskConstant.JOB_YARN_METRICS, metricsVo); - } else { - message.data(TaskConstant.JOB_YARNRESOURCE, null); - } - } else { - message.data(TaskConstant.JOB_YARNRESOURCE, null); - } + buildYarnResource(jobRequest, metricsVo, message); message .data("progress", Math.abs(job.get().getProgress())) - .data("execID", id) + .data("execID", execID) .data("progressInfo", list); } } else { @@ -385,6 +495,60 @@ public Message progressWithResource(HttpServletRequest req, @PathVariable("id") return message; } + private void buildYarnResource( + JobRequest jobRequest, Map metricsVo, Message message) { + try { + Map metrics = jobRequest.getMetrics(); + if (metrics.containsKey(TaskConstant.JOB_YARNRESOURCE)) { + + HashMap resourceMap = + (HashMap) metrics.get(TaskConstant.JOB_YARNRESOURCE); + ArrayList resoureList = new ArrayList<>(12); + if (null != resourceMap && !resourceMap.isEmpty()) { + resourceMap.forEach( + (applicationId, resource) -> { + resoureList.add(new YarnResourceWithStatusVo(applicationId, resource)); + }); + metricsVo.put(TaskConstant.JOB_YARNRESOURCE, resoureList); + Optional cores = + resourceMap.values().stream() + .map(resource -> resource.getQueueCores()) + .reduce((x, y) -> x + y); + Optional memory = + resourceMap.values().stream() + .map(resource -> resource.queueMemory()) + .reduce((x, y) -> x + y); + float corePercent = 0.0f; + float memoryPercent = 0.0f; + if (cores.isPresent() && memory.isPresent()) { + corePercent = + cores.get().floatValue() / EntranceConfiguration.YARN_QUEUE_CORES_MAX().getValue(); + memoryPercent = + memory.get().floatValue() + / (EntranceConfiguration.YARN_QUEUE_MEMORY_MAX().getValue().longValue() + * 1024 + * 1024 + * 1024); + } + String coreRGB = RGBUtils.getRGB(corePercent); + String memoryRGB = RGBUtils.getRGB(memoryPercent); + metricsVo.put(TaskConstant.JOB_CORE_PERCENT, corePercent); + metricsVo.put(TaskConstant.JOB_MEMORY_PERCENT, memoryPercent); + metricsVo.put(TaskConstant.JOB_CORE_RGB, coreRGB); + metricsVo.put(TaskConstant.JOB_MEMORY_RGB, memoryRGB); + + message.data(TaskConstant.JOB_YARN_METRICS, metricsVo); + } else { + message.data(TaskConstant.JOB_YARNRESOURCE, null); + } + } else { + message.data(TaskConstant.JOB_YARNRESOURCE, null); + } + } catch (Exception e) { + logger.error("build yarnResource error", e); + } + } + private void setJobProgressInfos( List> list, JobProgressInfo jobProgressInfo) { Map map = new HashMap<>(); @@ -403,10 +567,78 @@ private void setJobProgressInfos( @Override @RequestMapping(path = "/{id}/log", method = RequestMethod.GET) public Message log(HttpServletRequest req, @PathVariable("id") String id) { - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "log realId: " + realId); - Option job = Option.apply(null); + ModuleUserUtils.getOperationUser(req, "get job log"); Message message = null; + int fromLine = 0; + int size = 100; + boolean distinctLevel = true; + String fromLineStr = req.getParameter("fromLine"); + String sizeStr = req.getParameter("size"); + if (StringUtils.isNotBlank(fromLineStr)) { + fromLine = Math.max(Integer.parseInt(fromLineStr), 0); + } + if (StringUtils.isNotBlank(sizeStr)) { + size = Integer.parseInt(sizeStr) >= 0 ? Integer.parseInt(sizeStr) : 10000; + } + String distinctLevelStr = req.getParameter("distinctLevel"); + if ("false".equals(distinctLevelStr)) { + distinctLevel = false; + } + + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/log"); + return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = + Message.error( + "The job you just executed has ended. This interface no longer provides a query. It is recommended that you download the log file for viewing.(您刚刚执行的job已经结束，本接口不再提供查询，建议您下载日志文件进行查看)"); + message.setMethod("/api/entrance/" + id + "/log"); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return customer log", id); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + String log = + LogUtils.generateInfo( + "The job will failover soon, please try again later.(job很快就会failover，请稍后再试)"); + Object retLog; + if (distinctLevel) { + String[] array = new String[4]; + array[2] = log; + array[3] = log; + retLog = new ArrayList(Arrays.asList(array)); + } else { + retLog = log; + } + message.data("log", retLog).data("execID", "").data("taskID", id).data("fromLine", 0); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } + + Option job = null; try { job = entranceServer.getJob(realId); } catch (final Throwable t) { @@ -416,27 +648,10 @@ public Message log(HttpServletRequest req, @PathVariable("id") String id) { message.setMethod("/api/entrance/" + id + "/log"); return message; } - if (job.isDefined()) { + if (job != null && job.isDefined()) { logger.debug("begin to get log for {}(开始获取 {} 的日志)", job.get().getId(), job.get().getId()); LogReader logReader = entranceServer.getEntranceContext().getOrCreateLogManager().getLogReader(realId); - int fromLine = 0; - int size = 100; - boolean distinctLevel = true; - if (req != null) { - String fromLineStr = req.getParameter("fromLine"); - String sizeStr = req.getParameter("size"); - if (StringUtils.isNotBlank(fromLineStr)) { - fromLine = Math.max(Integer.parseInt(fromLineStr), 0); - } - if (StringUtils.isNotBlank(sizeStr)) { - size = Integer.parseInt(sizeStr) >= 0 ? Integer.parseInt(sizeStr) : 10000; - } - String distinctLevelStr = req.getParameter("distinctLevel"); - if ("false".equals(distinctLevelStr)) { - distinctLevel = false; - } - } Object retLog = null; int retFromLine = 0; @@ -458,7 +673,7 @@ public Message log(HttpServletRequest req, @PathVariable("id") String id) { e); message = Message.ok(); message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); } catch (final IllegalArgumentException e) { logger.debug( "Failed to get log information for :{}(为 {} 获取日志失败)", @@ -467,7 +682,7 @@ public Message log(HttpServletRequest req, @PathVariable("id") String id) { e); message = Message.ok(); message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); return message; } catch (final Exception e1) { logger.debug( @@ -477,7 +692,7 @@ public Message log(HttpServletRequest req, @PathVariable("id") String id) { e1); message = Message.error("Failed to get log information(获取日志信息失败)"); message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); return message; } finally { if (null != logReader && job.get().isCompleted()) { @@ -486,7 +701,7 @@ public Message log(HttpServletRequest req, @PathVariable("id") String id) { } message = Message.ok(); message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", retLog).data("execID", id).data("fromLine", retFromLine + fromLine); + message.data("log", retLog).data("execID", execID).data("fromLine", retFromLine + fromLine); logger.debug("success to get log for {} (获取 {} 日志成功)", job.get().getId(), job.get().getId()); } else { message = @@ -514,7 +729,6 @@ public Message killJobs( JsonNode taskIDNode = jsonNode.get("taskIDList"); ArrayList waitToForceKill = new ArrayList<>(); String userName = ModuleUserUtils.getOperationUser(req, "killJobs"); - if (idNode.size() != taskIDNode.size()) { return Message.error( "The length of the ID list does not match the length of the TASKID list(id列表的长度与taskId列表的长度不一致)"); @@ -527,7 +741,7 @@ public Message killJobs( String id = idNode.get(i).asText(); Long taskID = taskIDNode.get(i).asLong(); String realId = ZuulEntranceUtils.parseExecID(id)[3]; - Option job = Option.apply(null); + Option job = null; try { job = entranceServer.getJob(realId); } catch (Exception e) { @@ -541,7 +755,7 @@ public Message killJobs( continue; } Message message = null; - if (job.isEmpty()) { + if (job == null || job.isEmpty()) { logger.warn("can not find a job in entranceServer, will force to kill it"); waitToForceKill.add(taskID); message = Message.ok("Forced Kill task (强制杀死任务)"); @@ -577,11 +791,12 @@ public Message killJobs( if (null != logListener) { logListener.onLogUpdate( entranceJob, - "Job " - + jobReq.getId() - + " was kill by user successfully(任务" - + jobReq.getId() - + "已成功取消)"); + LogUtils.generateInfo( + "Job " + + jobReq.getId() + + " was kill by user successfully(任务" + + jobReq.getId() + + "已成功取消)")); } this.entranceServer .getEntranceContext() @@ -609,7 +824,7 @@ public Message killJobs( @ApiOperation(value = "kill", notes = "kill", response = Message.class) @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "excute id"), + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "exec id"), @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = "task id") }) @Override @@ -618,23 +833,68 @@ public Message kill( HttpServletRequest req, @PathVariable("id") String id, @RequestParam(value = "taskID", required = false) Long taskID) { - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - String userName = ModuleUserUtils.getOperationUser(req, "kill task realId:" + realId); + String userName = ModuleUserUtils.getOperationUser(req, "kill job"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/kill"); + return message; + } - Option job = Option.apply(null); + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = Message.error("The job already completed. Do not support kill.(任务已经结束，不支持kill)"); + message.setMethod("/api/entrance/" + id + "/kill"); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, but now force kill", id); + // TODO If failover during force kill, the job status may change from Cancelled to Running + long taskId = Long.parseLong(id); + JobHistoryHelper.forceKill(taskId); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.data("execID", "").data("taskID", id); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } + + Option job = null; try { job = entranceServer.getJob(realId); } catch (Exception e) { logger.warn("can not find a job in entranceServer, will force to kill it", e); // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 + if (taskID == null || taskID <= 0) { + message = Message.error("Get job by ID error, kill failed.(获取job时发生异常，kill失败)"); + return message; + } JobHistoryHelper.forceKill(taskID); - Message message = Message.ok("Forced Kill task (强制杀死任务)"); + message = Message.ok("Forced Kill task (强制杀死任务)"); message.setMethod("/api/entrance/" + id + "/kill"); message.setStatus(0); return message; } - Message message = null; - if (job.isEmpty()) { + + if (job == null || job.isEmpty()) { logger.warn("can not find a job in entranceServer, will force to kill it"); // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 JobHistoryHelper.forceKill(taskID); @@ -660,8 +920,7 @@ public Message kill( job.get().kill(); message = Message.ok("Successfully killed the job(成功kill了job)"); message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(0); - message.data("execID", id); + message.data("execID", execID); // ensure the job's state is cancelled in database if (job.get() instanceof EntranceJob) { EntranceJob entranceJob = (EntranceJob) job.get(); diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 7558ab6dc2..94531cd5fe 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -20,13 +20,17 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.entrance.EntranceContext; import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.conf.EntranceConfiguration; import org.apache.linkis.entrance.conf.EntranceConfiguration$; import org.apache.linkis.entrance.constant.ServiceNameConsts; import org.apache.linkis.entrance.execute.EntranceJob; +import org.apache.linkis.entrance.job.EntranceExecutionJob; import org.apache.linkis.entrance.log.LogReader; import org.apache.linkis.governance.common.protocol.conf.EntranceInstanceConfRequest; import org.apache.linkis.rpc.Sender; +import org.apache.commons.io.IOUtils; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; @@ -94,13 +98,19 @@ private void shutdownEntrance(ContextClosedEvent event) { if (shutdownFlag) { logger.warn("event has been handled"); } else { + if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED()) { + logger.warn("Entrance exit to update and clean all ConsumeQueue task instances"); + updateAllNotExecutionTaskInstances(false); + } + logger.warn("Entrance exit to stop all job"); - EntranceJob[] allUndoneJobs = getAllUndoneTask(null); - if (null != allUndoneJobs) { - for (EntranceJob job : allUndoneJobs) { + EntranceJob[] allUndoneTask = getAllUndoneTask(null); + if (null != allUndoneTask) { + for (EntranceJob job : allUndoneTask) { job.onFailure( "Your job will be marked as canceled because the Entrance service restarted(因为Entrance服务重启，您的任务将被标记为取消)", null); + IOUtils.closeQuietly(((EntranceExecutionJob) job).getLogWriter().get()); } } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java new file mode 100644 index 0000000000..4e66da5cc3 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.server; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.conf.EntranceConfiguration; +import org.apache.linkis.entrance.constant.ServiceNameConsts; +import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; +import org.apache.linkis.entrance.utils.JobHistoryHelper; +import org.apache.linkis.governance.common.entity.job.JobRequest; +import org.apache.linkis.publicservice.common.lock.entity.CommonLock; +import org.apache.linkis.publicservice.common.lock.service.CommonLockService; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.queue.SchedulerEventState; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +import scala.Enumeration; +import scala.collection.Iterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component(ServiceNameConsts.ENTRANCE_FAILOVER_SERVER) +public class EntranceFailoverJobServer { + + private static final Logger logger = LoggerFactory.getLogger(EntranceFailoverJobServer.class); + + @Autowired private EntranceServer entranceServer; + + @Autowired private CommonLockService commonLockService; + + private static String ENTRANCE_FAILOVER_LOCK = "ENTRANCE_FAILOVER_LOCK"; + + private ScheduledExecutorService scheduledExecutor; + + private Future future; + + @PostConstruct + public void init() { + if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { + this.scheduledExecutor = + Executors.newSingleThreadScheduledExecutor( + Utils.threadFactory("Linkis-Failover-Scheduler-Thread-", true)); + failoverTask(); + } + } + + @EventListener + private void shutdownFailover(ContextClosedEvent event) { + if (future != null && !future.isDone()) { + future.cancel(true); + } + if (scheduledExecutor != null) { + scheduledExecutor.shutdown(); + logger.info("Entrance Failover Server exit!"); + } + } + + public void failoverTask() { + future = + scheduledExecutor.scheduleWithFixedDelay( + () -> { + EntranceSchedulerContext schedulerContext = + (EntranceSchedulerContext) + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext(); + + // entrance do not failover job when it is offline + if (schedulerContext.getOfflineFlag()) return; + + CommonLock commonLock = new CommonLock(); + commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); + Boolean locked = false; + try { + locked = commonLockService.lock(commonLock, 30 * 1000L); + if (!locked) return; + logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); + + // get all entrance server from eureka + ServiceInstance[] serviceInstances = + Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); + if (serviceInstances == null || serviceInstances.length <= 0) return; + + // serverInstance to map + Map serverInstanceMap = + Arrays.stream(serviceInstances) + .collect( + Collectors.toMap( + ServiceInstance::getInstance, + ServiceInstance::getRegistryTimestamp, + (k1, k2) -> k2)); + + // It is very important to avoid repeated execute job + // when failover self job, if self instance is empty, the job can be repeated + // execute + if (!serverInstanceMap.containsKey(Sender.getThisInstance())) { + logger.warn( + "server has just started and has not get self info, it does not failover"); + return; + } + + // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) + long expiredTimestamp = 0L; + if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { + expiredTimestamp = + System.currentTimeMillis() + - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); + } + + List jobRequests = + JobHistoryHelper.queryWaitForFailoverTask( + serverInstanceMap, + getUnCompleteStatus(), + expiredTimestamp, + EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); + if (jobRequests.isEmpty()) return; + List ids = + jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); + logger.info("success query failover jobs , job size: {}, ids: {}", ids.size(), ids); + + // failover to local server + for (JobRequest jobRequest : jobRequests) { + entranceServer.failoverExecute(jobRequest); + } + logger.info("finished execute failover jobs, job ids: {}", ids); + + } catch (Exception e) { + logger.error("failover failed", e); + } finally { + if (locked) commonLockService.unlock(commonLock); + } + }, + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), + TimeUnit.MILLISECONDS); + } + + private List getUnCompleteStatus() { + List status = new ArrayList<>(); + Enumeration.ValueSet values = SchedulerEventState.values(); + Iterator iterator = values.iterator(); + while (iterator.hasNext()) { + Enumeration.Value next = iterator.next(); + if (!SchedulerEventState.isCompleted(next)) status.add(next.toString()); + } + return status; + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 1035de1e2b..b023065ee4 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -17,28 +17,45 @@ package org.apache.linkis.entrance +import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.exception.{ErrorException, LinkisException, LinkisRuntimeException} +import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.cs.CSEntranceHelper +import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} import org.apache.linkis.entrance.execute.EntranceJob -import org.apache.linkis.entrance.log.LogReader +import org.apache.linkis.entrance.job.EntranceExecutionJob +import org.apache.linkis.entrance.log.{Cache, CacheLogWriter, HDFSCacheLogWriter, LogReader} +import org.apache.linkis.entrance.parser.ParserUtils import org.apache.linkis.entrance.timeout.JobTimeoutManager import org.apache.linkis.entrance.utils.JobHistoryHelper +import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.protocol.task.RequestTaskKill import org.apache.linkis.governance.common.utils.LoggerUtils +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest +import org.apache.linkis.manager.label.entity.entrance.ExecuteOnceLabel import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.rpc.Sender +import org.apache.linkis.rpc.conf.RPCConfiguration import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} import org.apache.linkis.server.conf.ServerConfiguration +import org.apache.linkis.storage.utils.StorageUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils -import java.text.MessageFormat -import java.util +import org.springframework.beans.BeanUtils + +import java.{lang, util} +import java.text.{MessageFormat, SimpleDateFormat} +import java.util.Date + +import scala.collection.JavaConverters._ abstract class EntranceServer extends Logging { @@ -82,6 +99,283 @@ abstract class EntranceServer extends Logging { LoggerUtils.setJobIdMDC(jobRequest.getId.toString) val logAppender = new java.lang.StringBuilder() + jobRequest = dealInitedJobRequest(jobRequest, logAppender) + + val job = getEntranceContext.getOrCreateEntranceParser().parseToJob(jobRequest) + Utils.tryThrow { + job.init() + job.setLogListener(getEntranceContext.getOrCreateLogManager()) + job.setProgressListener(getEntranceContext.getOrCreatePersistenceManager()) + job.setJobListener(getEntranceContext.getOrCreatePersistenceManager()) + job match { + case entranceJob: EntranceJob => + entranceJob.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus) + case _ => + } + Utils.tryCatch { + if (logAppender.length() > 0) { + job.getLogListener.foreach(_.onLogUpdate(job, logAppender.toString.trim)) + } + } { t => + logger.error("Failed to write init log, reason: ", t) + } + + /** + * job.afterStateChanged() method is only called in job.run(), and job.run() is called only + * after job is scheduled so it suggest that we lack a hook for job init, currently we call + * this to trigger JobListener.onJobinit() + */ + Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) + getEntranceContext.getOrCreateScheduler().submit(job) + val msg = LogUtils.generateInfo( + s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted " + ) + logger.info(msg) + + job match { + case entranceJob: EntranceJob => + entranceJob.getJobRequest.setReqId(job.getId()) + if (jobTimeoutManager.timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) { + jobTimeoutManager.add(job.getId(), entranceJob) + } + entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) + case _ => + } + LoggerUtils.removeJobIdMDC() + job + } { t => + LoggerUtils.removeJobIdMDC() + job.onFailure("Submitting the query failed!(提交查询失败！)", t) + val _jobRequest: JobRequest = + getEntranceContext.getOrCreateEntranceParser().parseToJobRequest(job) + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(_jobRequest) + t match { + case e: LinkisException => e + case e: LinkisRuntimeException => e + case t: Throwable => + new SubmitFailedException( + SUBMITTING_QUERY_FAILED.getErrorCode, + SUBMITTING_QUERY_FAILED.getErrorDesc + ExceptionUtils.getRootCauseMessage(t), + t + ) + } + } + } + + def logReader(execId: String): LogReader + + def getJob(execId: String): Option[Job] = + getEntranceContext.getOrCreateScheduler().get(execId).map(_.asInstanceOf[Job]) + + private[entrance] def getEntranceWebSocketService: Option[EntranceWebSocketService] = + if (ServerConfiguration.BDP_SERVER_SOCKET_MODE.getValue) { + if (entranceWebSocketService.isEmpty) synchronized { + if (entranceWebSocketService.isEmpty) { + entranceWebSocketService = Some(new EntranceWebSocketService) + entranceWebSocketService.foreach(_.setEntranceServer(this)) + entranceWebSocketService.foreach( + getEntranceContext.getOrCreateEventListenerBus.addListener + ) + } + } + entranceWebSocketService + } else None + + def getAllUndoneTask(filterWords: String): Array[EntranceJob] = { + val consumers = getEntranceContext + .getOrCreateScheduler() + .getSchedulerContext + .getOrCreateConsumerManager + .listConsumers() + .toSet + val filterConsumer = if (StringUtils.isNotBlank(filterWords)) { + consumers.filter(_.getGroup.getGroupName.contains(filterWords)) + } else { + consumers + } + filterConsumer + .flatMap { consumer => + consumer.getRunningEvents ++ consumer.getConsumeQueue.getWaitingEvents + } + .filter(job => job != null && job.isInstanceOf[EntranceJob]) + .map(_.asInstanceOf[EntranceJob]) + .toArray + } + + def getAllConsumeQueueTask(): Array[EntranceJob] = { + val consumers = getEntranceContext + .getOrCreateScheduler() + .getSchedulerContext + .getOrCreateConsumerManager + .listConsumers() + .toSet + + consumers + .flatMap { consumer => + consumer.getConsumeQueue.getWaitingEvents + } + .filter(job => job != null && job.isInstanceOf[EntranceJob]) + .map(_.asInstanceOf[EntranceJob]) + .toArray + } + + def clearAllConsumeQueue(): Unit = { + getEntranceContext + .getOrCreateScheduler() + .getSchedulerContext + .getOrCreateConsumerManager + .listConsumers() + .foreach(_.getConsumeQueue.clearAll()) + } + + def updateAllNotExecutionTaskInstances(retryWhenUpdateFail: Boolean): Unit = { + val consumeQueueTasks = getAllConsumeQueueTask() + + clearAllConsumeQueue() + logger.info("Finished to clean all ConsumeQueue") + + if (consumeQueueTasks != null && consumeQueueTasks.length > 0) { + val taskIds = new util.ArrayList[Long]() + consumeQueueTasks.foreach(job => { + taskIds.add(job.getJobRequest.getId.asInstanceOf[Long]) + job match { + case entranceExecutionJob: EntranceExecutionJob => + val msg = LogUtils.generateWarn( + s"job ${job.getJobRequest.getId} clean from ConsumeQueue, wait for failover" + ) + entranceExecutionJob.getLogListener.foreach(_.onLogUpdate(entranceExecutionJob, msg)) + entranceExecutionJob.getLogWriter.foreach(_.close()) + case _ => + } + }) + + JobHistoryHelper.updateAllConsumeQueueTask(taskIds, retryWhenUpdateFail) + logger.info("Finished to update all not execution task instances") + } + } + + /** + * execute failover job (提交故障转移任务，返回新的execId) + * + * @param jobRequest + */ + def failoverExecute(jobRequest: JobRequest): Unit = { + + if (null == jobRequest || null == jobRequest.getId || jobRequest.getId <= 0) { + throw new EntranceErrorException( + PERSIST_JOBREQUEST_ERROR.getErrorCode, + PERSIST_JOBREQUEST_ERROR.getErrorDesc + ) + } + + val logAppender = new java.lang.StringBuilder() + logAppender.append( + "*************************************FAILOVER************************************** \n" + ) + + // try to kill ec + killOldEC(jobRequest, logAppender); + + // deal Inited jobRequest, if status is Inited, need to deal by all Interceptors, such as set log_path + if (SchedulerEventState.isInitedByStr(jobRequest.getStatus)) { + dealInitedJobRequest(jobRequest, logAppender) + } + + if ( + EntranceConfiguration.ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED.getValue && + SchedulerEventState.isRunningByStr(jobRequest.getStatus) + ) { + // deal Running jobRequest, if enabled, status changed from Running to Cancelled + dealRunningJobRequest(jobRequest, logAppender) + } else { + // init and submit + initAndSubmitJobRequest(jobRequest, logAppender) + } + } + + def killOldEC(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { + Utils.tryCatch { + logAppender.append( + LogUtils + .generateInfo(s"job ${jobRequest.getId} start to kill old ec \n") + ) + if ( + !SchedulerEventState.isRunning(SchedulerEventState.withName(jobRequest.getStatus)) + || !SchedulerEventState.isScheduled(SchedulerEventState.withName(jobRequest.getStatus)) + ) { + val msg = s"job ${jobRequest.getId} status is not running or scheduled, ignore it" + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") + return + } + + if ( + jobRequest.getMetrics == null + || !jobRequest.getMetrics.containsKey(TaskConstant.JOB_ENGINECONN_MAP) + ) { + val msg = s"job ${jobRequest.getId} not have EC info, ignore it" + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") + return + } + + val engineMap = jobRequest.getMetrics + .get(TaskConstant.JOB_ENGINECONN_MAP) + .asInstanceOf[util.Map[String, Object]] + + val engineInstance = + engineMap.asScala + .map(_._2.asInstanceOf[util.Map[String, Object]]) + .filter(_.containsKey(TaskConstant.ENGINE_INSTANCE)) + .maxBy(_.getOrDefault(TaskConstant.ENGINE_CONN_SUBMIT_TIME, "0").toString) + + if (engineInstance == null || engineInstance.containsKey(TaskConstant.FAILOVER_FLAG)) { + val msg = + s"job ${jobRequest.getId} do not submit to EC or already failover, not need kill ec" + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") + return + } + engineInstance.put(TaskConstant.FAILOVER_FLAG, "") + + val ecInstance = ServiceInstance( + GovernanceCommonConf.ENGINE_CONN_SPRING_NAME.getValue, + engineInstance.get(TaskConstant.ENGINE_INSTANCE).toString + ) + if (jobRequest.getLabels.asScala.exists(_.isInstanceOf[ExecuteOnceLabel])) { + // kill ec by linkismanager + val engineStopRequest = new EngineStopRequest + engineStopRequest.setServiceInstance(ecInstance) + // send to linkismanager kill ec + Sender + .getSender(RPCConfiguration.LINKIS_MANAGER_SERVICE_NAME.getValue) + .send(engineStopRequest) + val msg = + s"job ${jobRequest.getId} send EngineStopRequest to linkismanager, kill EC instance $ecInstance" + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") + } else if (engineInstance.containsKey(TaskConstant.ENGINE_CONN_TASK_ID)) { + // get ec taskId + val engineTaskId = engineInstance.get(TaskConstant.ENGINE_CONN_TASK_ID).toString + // send to ec kill task + Sender + .getSender(ecInstance) + .send(RequestTaskKill(engineTaskId)) + val msg = + s"job ${jobRequest.getId} send RequestTaskKill to kill engineConn $ecInstance, execID $engineTaskId" + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") + } + } { t => + logger.error(s"job ${jobRequest.getId} kill ec error", t) + } + } + + def dealInitedJobRequest(jobReq: JobRequest, logAppender: lang.StringBuilder): JobRequest = { + var jobRequest = jobReq Utils.tryThrow( getEntranceContext .getOrCreateEntranceInterceptors() @@ -128,6 +422,68 @@ abstract class EntranceServer extends Logging { .updateIfNeeded(jobRequest) error } + jobRequest + } + + def dealRunningJobRequest(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { + Utils.tryCatch { + // error_msg + val msg = + MessageFormat.format( + EntranceErrorCodeSummary.FAILOVER_RUNNING_TO_CANCELLED.getErrorDesc, + jobRequest.getId.toString + ) + // init jobRequest properties + jobRequest.setStatus(SchedulerEventState.Cancelled.toString) + jobRequest.setProgress("1.0") + jobRequest.setInstances(Sender.getThisInstance) + jobRequest.setErrorCode(EntranceErrorCodeSummary.FAILOVER_RUNNING_TO_CANCELLED.getErrorCode) + jobRequest.setErrorDesc(msg) + + // update jobRequest + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobRequest) + + // getOrGenerate log_path + var logPath = jobRequest.getLogPath + if (StringUtils.isBlank(logPath)) { + ParserUtils.generateLogPath(jobRequest, null) + logPath = jobRequest.getLogPath + logAppender.append( + LogUtils.generateInfo(s"job ${jobRequest.getId} generate new logPath $logPath \n") + ) + } + val job = getEntranceContext.getOrCreateEntranceParser().parseToJob(jobRequest) + val logWriter = getEntranceContext.getOrCreateLogManager().createLogWriter(job) + if (logAppender.length() > 0) { + logWriter.write(logAppender.toString.trim) + } + + logWriter.write(LogUtils.generateInfo(msg) + "\n") + logWriter.flush() + logWriter.close() + + } { case e: Exception => + logger.error(s"Job ${jobRequest.getId} failover, change status error", e) + } + } + + def initAndSubmitJobRequest(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { + // init properties + initJobRequestProperties(jobRequest, logAppender) + + // update jobRequest + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobRequest) + + // reset `UpdateOrderFlag` + jobRequest.setUpdateOrderFlag(true) + + logger.info(s"job ${jobRequest.getId} update JobRequest success") val job = getEntranceContext.getOrCreateEntranceParser().parseToJob(jobRequest) Utils.tryThrow { @@ -136,16 +492,16 @@ abstract class EntranceServer extends Logging { job.setProgressListener(getEntranceContext.getOrCreatePersistenceManager()) job.setJobListener(getEntranceContext.getOrCreatePersistenceManager()) job match { - case entranceJob: EntranceJob => + case entranceJob: EntranceJob => { entranceJob.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus) + } case _ => } Utils.tryCatch { - if (logAppender.length() > 0) { + if (logAppender.length() > 0) job.getLogListener.foreach(_.onLogUpdate(job, logAppender.toString.trim)) - } } { t => - logger.error("Failed to write init log, reason: ", t) + logger.error("Failed to write init JobRequest log, reason: ", t) } /** @@ -156,25 +512,21 @@ abstract class EntranceServer extends Logging { Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) getEntranceContext.getOrCreateScheduler().submit(job) val msg = LogUtils.generateInfo( - s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted " + s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted, success to failover" ) logger.info(msg) job match { case entranceJob: EntranceJob => entranceJob.getJobRequest.setReqId(job.getId()) - if (jobTimeoutManager.timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) { + if (jobTimeoutManager.timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) jobTimeoutManager.add(job.getId(), entranceJob) - } entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) case _ => } - LoggerUtils.removeJobIdMDC() - job } { t => - LoggerUtils.removeJobIdMDC() job.onFailure("Submitting the query failed!(提交查询失败！)", t) - val _jobRequest: JobRequest = + val _jobRequest = getEntranceContext.getOrCreateEntranceParser().parseToJobRequest(job) getEntranceContext .getOrCreatePersistenceManager() @@ -193,44 +545,80 @@ abstract class EntranceServer extends Logging { } } - def logReader(execId: String): LogReader + private def initJobRequestProperties( + jobRequest: JobRequest, + logAppender: lang.StringBuilder + ): Unit = { + logger.info(s"job ${jobRequest.getId} start to initialize the properties") + val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + val initInstance = Sender.getThisInstance + val initDate = new Date(System.currentTimeMillis) + val initStatus = SchedulerEventState.Inited.toString + val initProgress = "0.0" + val initReqId = "" - def getJob(execId: String): Option[Job] = - getEntranceContext.getOrCreateScheduler().get(execId).map(_.asInstanceOf[Job]) + logAppender.append( + LogUtils + .generateInfo(s"job ${jobRequest.getId} start to Initialize the properties \n") + ) + logAppender.append( + LogUtils.generateInfo(s"the instances ${jobRequest.getInstances} -> ${initInstance} \n") + ) + logAppender.append( + LogUtils.generateInfo( + s"the created_time ${sdf.format(jobRequest.getCreatedTime)} -> ${sdf.format(initDate)} \n" + ) + ) + logAppender.append( + LogUtils.generateInfo(s"the status ${jobRequest.getStatus} -> $initStatus \n") + ) + logAppender.append( + LogUtils.generateInfo(s"the progress ${jobRequest.getProgress} -> $initProgress \n") + ) - private[entrance] def getEntranceWebSocketService: Option[EntranceWebSocketService] = - if (ServerConfiguration.BDP_SERVER_SOCKET_MODE.getValue) { - if (entranceWebSocketService.isEmpty) synchronized { - if (entranceWebSocketService.isEmpty) { - entranceWebSocketService = Some(new EntranceWebSocketService) - entranceWebSocketService.foreach(_.setEntranceServer(this)) - entranceWebSocketService.foreach( - getEntranceContext.getOrCreateEventListenerBus.addListener + val metricMap = new util.HashMap[String, Object]() + if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_METRIC_ENGINE_CONN_ENABLED.getValue) { + if ( + jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( + TaskConstant.JOB_ENGINECONN_MAP ) - } + ) { + val oldEngineconnMap = jobRequest.getMetrics + .get(TaskConstant.JOB_ENGINECONN_MAP) + .asInstanceOf[util.Map[String, Object]] + metricMap.put(TaskConstant.JOB_ENGINECONN_MAP, oldEngineconnMap) } - entranceWebSocketService - } else None - - def getAllUndoneTask(filterWords: String): Array[EntranceJob] = { - val consumers = getEntranceContext - .getOrCreateScheduler() - .getSchedulerContext - .getOrCreateConsumerManager - .listConsumers() - .toSet - val filterConsumer = if (StringUtils.isNotBlank(filterWords)) { - consumers.filter(_.getGroup.getGroupName.contains(filterWords)) - } else { - consumers } - filterConsumer - .flatMap { consumer => - consumer.getRunningEvents ++ consumer.getConsumeQueue.getWaitingEvents + + if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_METRIC_YARN_RESOURCE_ENABLED.getValue) { + if ( + jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( + TaskConstant.JOB_YARNRESOURCE + ) + ) { + val oldResourceMap = jobRequest.getMetrics + .get(TaskConstant.JOB_YARNRESOURCE) + .asInstanceOf[util.Map[String, Object]] + metricMap.put(TaskConstant.JOB_YARNRESOURCE, oldResourceMap) } - .filter(job => job != null && job.isInstanceOf[EntranceJob]) - .map(_.asInstanceOf[EntranceJob]) - .toArray + } + + jobRequest.setInstances(initInstance) + jobRequest.setCreatedTime(initDate) + jobRequest.setStatus(initStatus) + jobRequest.setProgress(initProgress) + jobRequest.setReqId(initReqId) + jobRequest.setErrorCode(0) + jobRequest.setErrorDesc("") + jobRequest.setMetrics(metricMap) + jobRequest.getMetrics.put(TaskConstant.JOB_SUBMIT_TIME, initDate) + // Allow task status updates to be unordered + jobRequest.setUpdateOrderFlag(false) + + logAppender.append( + LogUtils.generateInfo(s"job ${jobRequest.getId} success to initialize the properties \n") + ) + logger.info(s"job ${jobRequest.getId} success to initialize the properties") } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 7c3935e69b..e053d3793c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -223,4 +223,44 @@ object EntranceConfiguration { val ENABLE_ENTRANCE_DIRTY_DATA_CLEAR = CommonVars("linkis.entrance.auto.clean.dirty.data.enable", false) + val ENTRANCE_FAILOVER_ENABLED = CommonVars("linkis.entrance.failover.enable", true).getValue + + val ENTRANCE_FAILOVER_SCAN_INIT_TIME = + CommonVars("linkis.entrance.failover.scan.init.time", 3 * 1000).getValue + + val ENTRANCE_FAILOVER_SCAN_INTERVAL = + CommonVars("linkis.entrance.failover.scan.interval", 30 * 1000).getValue + + val ENTRANCE_FAILOVER_DATA_NUM_LIMIT = + CommonVars("linkis.entrance.failover.data.num.limit", 10).getValue + + val ENTRANCE_FAILOVER_DATA_INTERVAL_TIME = + CommonVars("linkis.entrance.failover.data.interval.time", new TimeType("1d").toLong).getValue + + // if true, the waitForRetry job in runningJobs can be failover + val ENTRANCE_FAILOVER_RETRY_JOB_ENABLED = + CommonVars("linkis.entrance.failover.retry.job.enable", false) + + val ENTRANCE_UPDATE_BATCH_SIZE = CommonVars("linkis.entrance.update.batch.size", 100) + + // if true, the job in ConsumeQueue can be failover + val ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED = + CommonVars("linkis.entrance.shutdown.failover.consume.queue.enable", true).getValue + + val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", true) + + val ENTRANCE_GROUP_SCAN_INIT_TIME = CommonVars("linkis.entrance.group.scan.init.time", 3 * 1000) + + val ENTRANCE_GROUP_SCAN_INTERVAL = CommonVars("linkis.entrance.group.scan.interval", 60 * 1000) + + val ENTRANCE_FAILOVER_RETAIN_METRIC_ENGINE_CONN_ENABLED = + CommonVars("linkis.entrance.failover.retain.metric.engine.conn.enable", false) + + val ENTRANCE_FAILOVER_RETAIN_METRIC_YARN_RESOURCE_ENABLED = + CommonVars("linkis.entrance.failover.retain.metric.yarn.resource.enable", false) + + // if true, job whose status is running will be set to Cancelled + val ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED = + CommonVars("linkis.entrance.failover.running.kill.enable", false) + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala index 7a7cb7463a..a40c3fa35d 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala @@ -63,7 +63,7 @@ object CustomVariableUtils extends Logging { } val variableMap = TaskUtils .getVariableMap(jobRequest.getParams) - .asInstanceOf[util.HashMap[String, String]] + .asInstanceOf[util.Map[String, String]] variables.putAll(variableMap) if (!variables.containsKey("user")) { variables.put("user", jobRequest.getExecuteUser) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala index 573c134493..653e9ad78b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala @@ -67,7 +67,7 @@ object UserCreatorIPCheckUtils extends Logging { def checkUserIp(jobRequest: JobRequest, logAppender: lang.StringBuilder): JobRequest = { // Get IP address - val jobIp = jobRequest.getSource.get(TaskConstant.REQUEST_IP) + val jobIp = jobRequest.getSource.getOrDefault(TaskConstant.REQUEST_IP, "") logger.debug(s"start to checkTenantLabel $jobIp") if (StringUtils.isNotBlank(jobIp)) { jobRequest match { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala new file mode 100644 index 0000000000..e2f0ab1d5a --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler + +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.job.EntranceExecutionJob +import org.apache.linkis.entrance.utils.JobHistoryHelper +import org.apache.linkis.scheduler.SchedulerContext +import org.apache.linkis.scheduler.queue.Group +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer + +import java.util +import java.util.concurrent.ExecutorService + +import scala.collection.JavaConverters.collectionAsScalaIterableConverter + +class EntranceFIFOUserConsumer( + schedulerContext: SchedulerContext, + executeService: ExecutorService, + private var group: Group +) extends FIFOUserConsumer(schedulerContext, executeService, group) { + + override def loop(): Unit = { + // When offlineFlag=true, the unsubmitted tasks will be failover, and the running tasks will wait for completion. + // In this case,super.loop only submits the retry task, but the retry task can failover and speed up the entrance offline + // (当offlineFlag=true时，未提交任务会被故障转移，运行中任务会等待完成.此时super.loop只会提交重试任务，但是重试任务完全可以故障转移，加快entrance下线) + schedulerContext match { + case entranceSchedulerContext: EntranceSchedulerContext => + if ( + entranceSchedulerContext.getOfflineFlag && EntranceConfiguration.ENTRANCE_FAILOVER_RETRY_JOB_ENABLED.getValue + ) { + val jobs = scanAllRetryJobsAndRemove() + if (!jobs.isEmpty) { + val ids = new util.ArrayList[Long]() + jobs.asScala.foreach { + case entranceJob: EntranceExecutionJob => + entranceJob.getLogWriter.foreach(_.close()) + ids.add(entranceJob.getJobRequest.getId) + case _ => + } + JobHistoryHelper.updateBatchInstancesEmpty(ids) + } + Utils.tryQuietly(Thread.sleep(5000)) + return + } + case _ => + } + + // general logic + super.loop() + + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index 7f16dd2463..0f31351b48 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -17,7 +17,6 @@ package org.apache.linkis.entrance.scheduler -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.conf.{CommonVars, Configuration} import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration @@ -28,16 +27,12 @@ import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } -import org.apache.linkis.instance.label.client.InstanceLabelClient -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{ ConcurrentEngineConnLabel, EngineTypeLabel, UserCreatorLabel } -import org.apache.linkis.manager.label.entity.route.RouteLabel import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.utils.TaskUtils @@ -157,40 +152,10 @@ class EntranceGroupFactory extends GroupFactory with Logging { } private def getUserMaxRunningJobs(keyAndValue: util.Map[String, String]): Int = { - var userDefinedRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) - var entranceNum = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName).length - val labelList = new util.ArrayList[Label[_]]() - val offlineRouteLabel = LabelBuilderFactoryContext.getLabelBuilderFactory - .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) - labelList.add(offlineRouteLabel) - var offlineIns: Array[ServiceInstance] = null - Utils.tryAndWarn { - offlineIns = InstanceLabelClient.getInstance - .getInstanceFromLabel(labelList) - .asScala - .filter(l => - null != l && l.getApplicationName - .equalsIgnoreCase(Sender.getThisServiceInstance.getApplicationName) - ) - .toArray - } - if (null != offlineIns) { - logger.info(s"There are ${offlineIns.length} offlining instance.") - entranceNum = entranceNum - offlineIns.length - } - /* - Sender.getInstances may get 0 instances due to cache in Sender. So this instance is the one instance. - */ - if (0 >= entranceNum) { - logger.error( - s"Got ${entranceNum} ${Sender.getThisServiceInstance.getApplicationName} instances." - ) - entranceNum = 1 - } Math.max( EntranceConfiguration.ENTRANCE_INSTANCE_MIN.getValue, - userDefinedRunningJobs / entranceNum - ); + EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) + ) } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala new file mode 100644 index 0000000000..2cdee97cc7 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.instance.label.client.InstanceLabelClient +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.rpc.Sender +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer +import org.apache.linkis.scheduler.queue.parallelqueue.{ParallelConsumerManager, ParallelGroup} + +import java.util +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ + +class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: String) + extends ParallelConsumerManager(maxParallelismUsers, schedulerName) { + + override protected def createConsumer(groupName: String): FIFOUserConsumer = { + val group = getSchedulerContext.getOrCreateGroupFactory.getGroup(groupName) + new EntranceFIFOUserConsumer(getSchedulerContext, getOrCreateExecutorService, group) + } + + if (EntranceConfiguration.ENTRANCE_GROUP_SCAN_ENABLED.getValue) { + Utils.defaultScheduler.scheduleAtFixedRate( + new Runnable { + override def run(): Unit = { + Utils.tryAndError { + logger.info("start refresh consumer group maxAllowRunningJobs") + // get all entrance server from eureka + val serviceInstances = + Sender.getInstances(Sender.getThisServiceInstance.getApplicationName) + if (null == serviceInstances || serviceInstances.isEmpty) return + + // get all offline label server + val routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory + .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) + val labels = new util.ArrayList[Label[_]] + labels.add(routeLabel) + val labelInstances = InstanceLabelClient.getInstance.getInstanceFromLabel(labels) + + // get active entrance server + val allInstances = new util.ArrayList[ServiceInstance]() + allInstances.addAll(serviceInstances.toList.asJava) + allInstances.removeAll(labelInstances) + // refresh all group maxAllowRunningJobs + refreshAllGroupMaxAllowRunningJobs(allInstances.size()) + logger.info("Finished to refresh consumer group maxAllowRunningJobs") + } + } + }, + EntranceConfiguration.ENTRANCE_GROUP_SCAN_INIT_TIME.getValue, + EntranceConfiguration.ENTRANCE_GROUP_SCAN_INTERVAL.getValue, + TimeUnit.MILLISECONDS + ) + } + + def refreshAllGroupMaxAllowRunningJobs(validInsCount: Int): Unit = { + listConsumers() + .foreach(item => { + item.getGroup match { + case group: ParallelGroup => + val maxAllowRunningJobs = Math.round(group.getMaxRunningJobs / validInsCount) + group.setMaxAllowRunningJobs(maxAllowRunningJobs) + logger + .info( + "group {} refresh maxAllowRunningJobs => {}/{}={}", + Array( + group.getGroupName, + group.getMaxRunningJobs.toString, + validInsCount.toString, + maxAllowRunningJobs.toString + ): _* + ) + case _ => + } + }) + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala index d5de2cc2da..1638b0fb1c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala @@ -28,6 +28,11 @@ class EntranceSchedulerContext extends SchedulerContext { private var consumerManager: ConsumerManager = _ private var executorManager: ExecutorManager = _ + private var offlineFlag: Boolean = false + + def setOfflineFlag(offlineFlag: Boolean): Unit = this.offlineFlag = offlineFlag + def getOfflineFlag: Boolean = this.offlineFlag + def this( groupFactory: GroupFactory, consumerManager: ConsumerManager, diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index ec29128889..44e2357b34 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -69,6 +69,11 @@ object JobHistoryHelper extends Logging { else task.getStatus } + def getProgressByTaskID(taskID: Long): String = { + val task = getTaskByTaskID(taskID) + if (task == null) "0" else task.getProgress + } + def getRequestIpAddr(req: HttpServletRequest): String = { val addrList = List( Option(req.getHeader("x-forwarded-for")).getOrElse("").split(",")(0), @@ -123,7 +128,144 @@ object JobHistoryHelper extends Logging { sender.ask(jobReqBatchUpdate) } - private def getTaskByTaskID(taskID: Long): JobRequest = { + /** + * Get all consume queue task and batch update instances(获取所有消费队列中的任务进行批量更新) + * + * @param taskIdList + * @param retryWhenUpdateFail + */ + def updateAllConsumeQueueTask( + taskIdList: util.List[Long], + retryWhenUpdateFail: Boolean = false + ): Unit = { + + if (taskIdList.isEmpty) return + + val updateTaskIds = new util.ArrayList[Long]() + + if ( + EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && + taskIdList.size() > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue + ) { + for (i <- 0 until EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { + updateTaskIds.add(taskIdList.get(i)) + } + } else { + updateTaskIds.addAll(taskIdList) + } + val list = new util.ArrayList[Long]() + list.addAll(taskIdList) + try { + val successTaskIds = updateBatchInstancesEmpty(updateTaskIds) + if (retryWhenUpdateFail) { + list.removeAll(successTaskIds) + } else { + list.removeAll(updateTaskIds) + } + } catch { + case e: Exception => + logger.warn("update batch instances failed, wait for retry", e) + Thread.sleep(1000) + } + updateAllConsumeQueueTask(list, retryWhenUpdateFail) + + } + + /** + * Batch update instances(批量更新instances字段) + * + * @param taskIdList + * @return + */ + def updateBatchInstancesEmpty(taskIdList: util.List[Long]): util.List[Long] = { + val jobReqList = new util.ArrayList[JobRequest]() + taskIdList.asScala.foreach(taskID => { + val jobRequest = new JobRequest + jobRequest.setId(taskID) + jobRequest.setInstances("") + jobReqList.add(jobRequest) + }) + val jobReqBatchUpdate = JobReqBatchUpdate(jobReqList) + Utils.tryCatch { + val response = sender.ask(jobReqBatchUpdate) + response match { + case resp: util.List[JobRespProtocol] => + // todo filter success data, rpc have bug +// resp.asScala +// .filter(r => +// r.getStatus == SUCCESS_FLAG && r.getData.containsKey(JobRequestConstants.JOB_ID) +// ) +// .map(_.getData.get(JobRequestConstants.JOB_ID).asInstanceOf[java.lang.Long]) +// .toList + + taskIdList + case _ => + throw JobHistoryFailedException( + "update batch instances from jobhistory not a correct List type" + ) + } + } { + case errorException: ErrorException => throw errorException + case e: Exception => + val e1 = + JobHistoryFailedException( + s"update batch instances ${taskIdList.asScala.mkString(",")} error" + ) + e1.initCause(e) + throw e + } + } + + /** + * query wait for failover task(获取待故障转移的任务) + * + * @param reqMap + * @param statusList + * @param startTimestamp + * @param limit + * @return + */ + def queryWaitForFailoverTask( + reqMap: util.Map[String, java.lang.Long], + statusList: util.List[String], + startTimestamp: Long, + limit: Int + ): util.List[JobRequest] = { + val requestFailoverJob = RequestFailoverJob(reqMap, statusList, startTimestamp, limit) + val tasks = Utils.tryCatch { + val response = sender.ask(requestFailoverJob) + response match { + case responsePersist: JobRespProtocol => + val status = responsePersist.getStatus + if (status != SUCCESS_FLAG) { + logger.error(s"query from jobHistory status failed, status is $status") + throw JobHistoryFailedException("query from jobHistory status failed") + } + val data = responsePersist.getData + data.get(JobRequestConstants.JOB_HISTORY_LIST) match { + case tasks: List[JobRequest] => + tasks.asJava + case _ => + throw JobHistoryFailedException( + s"query from jobhistory not a correct List type, instances ${reqMap.keySet()}" + ) + } + case _ => + logger.error("get query response incorrectly") + throw JobHistoryFailedException("get query response incorrectly") + } + } { + case errorException: ErrorException => throw errorException + case e: Exception => + val e1 = + JobHistoryFailedException(s"query failover task error, instances ${reqMap.keySet()} ") + e1.initCause(e) + throw e + } + tasks + } + + def getTaskByTaskID(taskID: Long): JobRequest = { val jobRequest = new JobRequest jobRequest.setId(taskID) jobRequest.setSource(null) @@ -176,15 +318,15 @@ object JobHistoryHelper extends Logging { val ecResourceMap = if (resourceInfo == null) new util.HashMap[String, ResourceWithStatus] else resourceInfo if (resourceMap != null) { - resourceMap.asInstanceOf[util.HashMap[String, ResourceWithStatus]].putAll(ecResourceMap) + resourceMap.asInstanceOf[util.Map[String, ResourceWithStatus]].putAll(ecResourceMap) } else { metricsMap.put(TaskConstant.JOB_YARNRESOURCE, ecResourceMap) } - var engineInstanceMap: util.HashMap[String, AnyRef] = null + var engineInstanceMap: util.Map[String, AnyRef] = null if (metricsMap.containsKey(TaskConstant.JOB_ENGINECONN_MAP)) { engineInstanceMap = metricsMap .get(TaskConstant.JOB_ENGINECONN_MAP) - .asInstanceOf[util.HashMap[String, AnyRef]] + .asInstanceOf[util.Map[String, AnyRef]] } else { engineInstanceMap = new util.HashMap[String, AnyRef]() metricsMap.put(TaskConstant.JOB_ENGINECONN_MAP, engineInstanceMap) @@ -194,7 +336,7 @@ object JobHistoryHelper extends Logging { val ticketId = infoMap.get(TaskConstant.TICKET_ID).asInstanceOf[String] val engineExtraInfoMap = engineInstanceMap .getOrDefault(ticketId, new util.HashMap[String, AnyRef]) - .asInstanceOf[util.HashMap[String, AnyRef]] + .asInstanceOf[util.Map[String, AnyRef]] engineExtraInfoMap.putAll(infoMap) engineInstanceMap.put(ticketId, engineExtraInfoMap) } else { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java index 392f4dd097..c12e2791b3 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java @@ -48,9 +48,7 @@ public class UJESSQLDriver extends UJESSQLDriverMain implements Driver { static String TOKEN_VALUE = "value"; static String PASSWORD = "password"; static boolean TABLEAU_SERVER = false; - static String LIMIT_ENABLED = "true"; - static String LIMIT = "limit"; - + static String FIXED_SESSION = "fixedSession"; static String VERSION = "version"; static int DEFAULT_VERSION = 1; static String MAX_CONNECTION_SIZE = "maxConnectionSize"; diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java index 8f7dcfed12..3e1e7e3182 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java @@ -17,10 +17,16 @@ package org.apache.linkis.ujes.jdbc.utils; +import org.apache.linkis.common.utils.Utils; + +import java.util.concurrent.atomic.AtomicInteger; + public class JDBCUtils { private static final char SEARCH_STRING_ESCAPE = '\\'; + public static final AtomicInteger idCreator = new AtomicInteger(); + public static String convertPattern(final String pattern) { if (pattern == null) { return ".*"; @@ -50,4 +56,8 @@ public static String convertPattern(final String pattern) { return result.toString(); } } + + public static String getUniqId() { + return Utils.getLocalHostname() + "_" + JDBCUtils.idCreator.getAndIncrement(); + } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index b800698766..7b3e2ada8c 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -18,6 +18,7 @@ package org.apache.linkis.ujes.jdbc import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel, RunType} import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator @@ -25,6 +26,7 @@ import org.apache.linkis.ujes.client.UJESClient import org.apache.linkis.ujes.client.request.JobSubmitAction import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain._ +import org.apache.linkis.ujes.jdbc.utils.JDBCUtils import org.apache.commons.lang3.StringUtils @@ -99,6 +101,18 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope private[jdbc] val serverURL = props.getProperty("URL") + private[jdbc] val fixedSessionEnabled = + if ( + props + .containsKey(FIXED_SESSION) && "true".equalsIgnoreCase(props.getProperty(FIXED_SESSION)) + ) { + true + } else { + false + } + + private val connectionId = JDBCUtils.getUniqId() + private val labelMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] private val startupParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] @@ -113,7 +127,14 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope if (params != null & params.length() > 0) { params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => - return EngineTypeLabelCreator.createEngineTypeLabel(v) + if (v.contains('-')) { + val factory = LabelBuilderFactoryContext.getLabelBuilderFactory + val label = factory.createLabel(classOf[EngineTypeLabel]) + label.setStringValue(v) + return label + } else { + return EngineTypeLabelCreator.createEngineTypeLabel(v) + } case _ => } } @@ -429,8 +450,10 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope val runType = EngineType.mapStringToEngineType(engine) match { case EngineType.SPARK => RunType.SQL case EngineType.HIVE => RunType.HIVE + case EngineType.REPL => RunType.REPL case EngineType.TRINO => RunType.TRINO_SQL case EngineType.PRESTO => RunType.PRESTO_SQL + case EngineType.NEBULA => RunType.NEBULA_SQL case EngineType.ELASTICSEARCH => RunType.ES_SQL case EngineType.JDBC => RunType.JDBC case EngineType.PYTHON => RunType.SHELL @@ -444,6 +467,10 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope labelMap.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue) labelMap.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, s"$user-$creator") labelMap.put(LabelKeyConstant.CODE_TYPE_KEY, engineToCodeType(engineTypeLabel.getEngineType)) + if (fixedSessionEnabled) { + labelMap.put(LabelKeyConstant.FIXED_EC_KEY, connectionId) + logger.info("Fixed session is enable session id is {}", connectionId) + } val jobSubmitAction = JobSubmitAction.builder .addExecuteCode(code) @@ -462,4 +489,6 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope result } + override def toString: String = "LinkisConnection_" + connectionId + } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala index 517f8b07ae..bea24181a3 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala @@ -66,7 +66,6 @@ object UJESClientFactory extends Logging { clientConfigBuilder.setAuthenticationStrategy(new StaticAuthenticationStrategy()) clientConfigBuilder.readTimeout(100000) clientConfigBuilder.maxConnectionSize(20) - clientConfigBuilder.readTimeout(10000) val params = props.getProperty(PARAMS) var versioned = false if (StringUtils.isNotBlank(params)) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala index 6296f399e5..ab2f6dda10 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala @@ -29,10 +29,10 @@ import java.sql.{ DriverPropertyInfo, SQLFeatureNotSupportedException } -import java.util.Properties +import java.util.{Locale, Properties} import java.util.logging.Logger -import scala.collection.JavaConversions +import scala.collection.JavaConverters._ class UJESSQLDriverMain extends Driver with Logging { @@ -72,9 +72,8 @@ class UJESSQLDriverMain extends Driver with Logging { case Array(TOKEN_VALUE, value) => props.setProperty(TOKEN_VALUE, value) false - case Array(LIMIT, value) => - props.setProperty(LIMIT, value) - UJESSQLDriverMain.LIMIT_ENABLED = value.toLowerCase() + case Array(FIXED_SESSION, value) => + props.setProperty(FIXED_SESSION, value) false case Array(key, _) => if (StringUtils.isBlank(key)) { @@ -138,8 +137,7 @@ object UJESSQLDriverMain { val TOKEN_VALUE = UJESSQLDriver.TOKEN_VALUE val PASSWORD = UJESSQLDriver.PASSWORD val TABLEAU_SERVER = UJESSQLDriver.TABLEAU_SERVER - val LIMIT = UJESSQLDriver.LIMIT - var LIMIT_ENABLED = UJESSQLDriver.LIMIT_ENABLED + val FIXED_SESSION = UJESSQLDriver.FIXED_SESSION val VERSION = UJESSQLDriver.VERSION val DEFAULT_VERSION = UJESSQLDriver.DEFAULT_VERSION @@ -157,8 +155,7 @@ object UJESSQLDriverMain { connectionParams: String, variableMap: java.util.Map[String, Any] ): String = { - val variables = JavaConversions - .mapAsScalaMap(variableMap) + val variables = variableMap.asScala .map(kv => VARIABLE_HEADER + kv._1 + KV_SPLIT + kv._2) .mkString(PARAM_SPLIT) if (StringUtils.isNotBlank(connectionParams)) connectionParams + PARAM_SPLIT + variables @@ -186,17 +183,20 @@ object UJESSQLDriverMain { ): String = { val sb = new StringBuilder if (StringUtils.isNotBlank(version)) sb.append(VERSION).append(KV_SPLIT).append(version) - if (maxConnectionSize > 0) + if (maxConnectionSize > 0) { sb.append(PARAM_SPLIT).append(MAX_CONNECTION_SIZE).append(KV_SPLIT).append(maxConnectionSize) - if (readTimeout > 0) + } + if (readTimeout > 0) { sb.append(PARAM_SPLIT).append(READ_TIMEOUT).append(KV_SPLIT).append(readTimeout) + } if (enableDiscovery) { sb.append(PARAM_SPLIT).append(ENABLE_DISCOVERY).append(KV_SPLIT).append(enableDiscovery) - if (enableLoadBalancer) + if (enableLoadBalancer) { sb.append(PARAM_SPLIT) .append(ENABLE_LOADBALANCER) .append(KV_SPLIT) .append(enableLoadBalancer) + } } if (sb.startsWith(PARAM_SPLIT)) sb.toString.substring(PARAM_SPLIT.length) else sb.toString } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala index 5df56733e4..0ed47925c6 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala @@ -193,7 +193,7 @@ class UJESSQLResultSet( if (metaData == null) init() currentRowCursor += 1 if (null == resultSetRow || currentRowCursor > resultSetRow.size() - 1) { - if (UJESSQLDriverMain.LIMIT_ENABLED.equals("false") && !isCompleted) { + if (!isCompleted) { updateResultSet() if (isCompleted) { return false @@ -242,7 +242,7 @@ class UJESSQLResultSet( } else { dataType.toLowerCase(Locale.getDefault) match { case null => throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) - case "string" => value.toString + case "char" | "varchar" | "nvarchar" | "string" => value case "short" => value.toShort case "int" => value.toInt case "long" => value.toLong @@ -250,11 +250,8 @@ class UJESSQLResultSet( case "double" => value.toDouble case "boolean" => value.toBoolean case "byte" => value.toByte - case "char" => value.toString - case "timestamp" => value.toString - case "varchar" => value.toString - case "nvarchar" => value.toString - case "date" => value.toString + case "timestamp" => value + case "date" => value case "bigint" => value.toLong case "decimal" => value.toDouble case "array" => value.toArray @@ -974,8 +971,9 @@ class UJESSQLResultSet( logger.info(s"the value of value is $value and the value of localTimeZone is $localTimeZone") if (wasNull()) { null - } else + } else { new Timestamp(TIMESTAMP_FORMATTER.withZone(localTimeZone).parseMillis(String.valueOf(value))) + } } override def getTimestamp(columnIndex: Int, cal: Calendar): Timestamp = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala index ed8936cc77..c7de7d3734 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala @@ -34,8 +34,7 @@ object JDBCDriverPreExecutionHook extends Logging { val hooks = new ArrayBuffer[JDBCDriverPreExecutionHook]() CommonVars( "wds.linkis.jdbc.pre.hook.class", - "org.apache.linkis.ujes.jdbc.hook.impl.TableauPreExecutionHook," + - "org.apache.linkis.ujes.jdbc.hook.impl.NoLimitExecutionHook" + "org.apache.linkis.ujes.jdbc.hook.impl.TableauPreExecutionHook" ).getValue.split(",") foreach { hookStr => Utils.tryCatch { val clazz = Class.forName(hookStr.trim) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java index d916387d29..0f018ca9de 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -68,7 +68,8 @@ public class AMConfiguration { public static final CommonVars MULTI_USER_ENGINE_TYPES = CommonVars.apply( - "wds.linkis.multi.user.engine.types", "jdbc,es,presto,io_file,appconn,openlookeng,trino"); + "wds.linkis.multi.user.engine.types", + "jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase"); public static final CommonVars ALLOW_BATCH_KILL_ENGINE_TYPES = CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); @@ -104,8 +105,8 @@ public class AMConfiguration { public static String getDefaultMultiEngineUser() { String jvmUser = Utils.getJvmUser(); return String.format( - "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", io_file:\"root\"}", - jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); + "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\", hbase:\"%s\",io_file:\"root\"}", + jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); } public static boolean isMultiUserEngine(String engineType) { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.java index 8e86f6906a..ad259ec30c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.java @@ -53,10 +53,10 @@ private Comparator sortByResource() { RMNode nodeBRm = (RMNode) nodeB; if (nodeARm.getNodeResource() == null || nodeARm.getNodeResource().getLeftResource() == null) { - return -1; + return 1; } else if (nodeBRm.getNodeResource() == null || nodeBRm.getNodeResource().getLeftResource() == null) { - return 1; + return -1; } else { if (nodeARm .getNodeResource() @@ -69,7 +69,11 @@ private Comparator sortByResource() { .moreThan(nodeBRm.getNodeResource().getLeftResource())) { return -1; } else { - return 1; + // 从大到小排序 (Sort from large to small) + return -(nodeARm + .getNodeResource() + .getLeftResource() + .compare(nodeBRm.getNodeResource().getLeftResource())); } } } catch (Throwable t) { @@ -93,10 +97,10 @@ private Comparator sortByResourceRate() { RMNode nodeBRm = (RMNode) nodeB; if (nodeARm.getNodeResource() == null || nodeARm.getNodeResource().getLeftResource() == null) { - return -1; + return 1; } else if (nodeBRm.getNodeResource() == null || nodeBRm.getNodeResource().getLeftResource() == null) { - return 1; + return -1; } else { float aRate = ResourceUtils.getLoadInstanceResourceRate( @@ -106,7 +110,7 @@ private Comparator sortByResourceRate() { ResourceUtils.getLoadInstanceResourceRate( nodeBRm.getNodeResource().getLeftResource(), nodeBRm.getNodeResource().getMaxResource()); - return Float.compare(aRate, bRate); + return -Float.compare(aRate, bRate); } } catch (Throwable t) { logger.warn("Failed to Compare resource " + t.getMessage()); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/conf/LabelManagerConf.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/conf/LabelManagerConf.java index f436254911..9aa5ff797f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/conf/LabelManagerConf.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/conf/LabelManagerConf.java @@ -23,4 +23,7 @@ public class LabelManagerConf { public static final String LONG_LIVED_LABEL = CommonVars.apply("wds.linkis.label.node.long.lived.label.keys", "tenant").getValue(); + + public static final boolean COMBINED_WITHOUT_YARN_DEFAULT = + CommonVars.apply("linkis.combined.without.yarn.default", true).getValue(); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java index 5bda339194..9d3140267b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java @@ -18,10 +18,13 @@ package org.apache.linkis.manager.rm.domain; import org.apache.linkis.governance.common.conf.GovernanceCommonConf; +import org.apache.linkis.manager.common.conf.RMConfiguration; import org.apache.linkis.manager.label.builder.CombinedLabelBuilder; +import org.apache.linkis.manager.label.conf.LabelManagerConf; import org.apache.linkis.manager.label.entity.CombinedLabel; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.ResourceLabel; +import org.apache.linkis.manager.label.entity.cluster.ClusterLabel; import org.apache.linkis.manager.label.entity.em.EMInstanceLabel; import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel; import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; @@ -49,7 +52,8 @@ public class RMLabelContainer { private EngineTypeLabel engineTypeLabel; private UserCreatorLabel userCreatorLabel; private EngineInstanceLabel engineInstanceLabel; - private CombinedLabel combinedUserCreatorEngineTypeLabel; + private ClusterLabel clusterLabel; + private CombinedLabel combinedResourceLabel; private Label currentLabel; public RMLabelContainer(List> labels) { @@ -57,14 +61,16 @@ public RMLabelContainer(List> labels) { this.lockedLabels = Lists.newArrayList(); try { if (getUserCreatorLabel() != null && getEngineTypeLabel() != null) { - this.combinedUserCreatorEngineTypeLabel = - (CombinedLabel) - combinedLabelBuilder.build( - "", Lists.newArrayList(getUserCreatorLabel(), getEngineTypeLabel())); - this.labels.add(combinedUserCreatorEngineTypeLabel); + List