diff --git a/tools/automatic_query_fixer/.gitattributes b/tools/automatic_query_fixer/.gitattributes new file mode 100644 index 000000000..00a51aff5 --- /dev/null +++ b/tools/automatic_query_fixer/.gitattributes @@ -0,0 +1,6 @@ +# +# https://help.github.com/articles/dealing-with-line-endings/ +# +# These are explicitly windows files and should use crlf +*.bat text eol=crlf + diff --git a/tools/automatic_query_fixer/.gitignore b/tools/automatic_query_fixer/.gitignore new file mode 100644 index 000000000..1fd477c43 --- /dev/null +++ b/tools/automatic_query_fixer/.gitignore @@ -0,0 +1,16 @@ +.gradle +**/build/ +!src/**/build/ +.idea + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties \ No newline at end of file diff --git a/tools/automatic_query_fixer/build.gradle b/tools/automatic_query_fixer/build.gradle new file mode 100644 index 000000000..83c06cf7d --- /dev/null +++ b/tools/automatic_query_fixer/build.gradle @@ -0,0 +1,45 @@ +plugins { + id 'java' + id 'application' +} + +group 'com.google.cloud.bigquery.utils' +version '1.0-SNAPSHOT' + + + +application { + mainClassName = "com.google.cloud.bigquery.utils.auto_query_fixer.Application" +} + +repositories { + mavenCentral() + flatDir { + dirs 'libs' + } + +} + +dependencies { + + compileOnly 'org.projectlombok:lombok:1.18.12' + annotationProcessor 'org.projectlombok:lombok:1.18.12' + implementation 'org.apache.calcite.avatica:avatica-core:1.17.0' + implementation "org.apache.commons:commons-lang3:3.1" + // google flogger dependencies + compile group: 'com.google.flogger', name: 'flogger', version: '0.1' + compile 'com.google.flogger:flogger-system-backend:0.4' + + // local jar + implementation name: 'calcite-babel-1.23.0-SNAPSHOT' + implementation name: 'calcite-core-1.23.0-SNAPSHOT' + implementation name: 'calcite-linq4j-1.23.0-SNAPSHOT' + //dependency from local jar + compile 'com.google.guava:guava:23.5-jre' + compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' + compile 'org.slf4j:slf4j-simple:1.7.25' + + testCompileOnly 'org.projectlombok:lombok:1.18.12' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + testCompile group: 'junit', name: 'junit', version: '4.12' +} diff --git a/tools/automatic_query_fixer/gradle/wrapper/gradle-wrapper.jar b/tools/automatic_query_fixer/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..f3d88b1c2 Binary files /dev/null and b/tools/automatic_query_fixer/gradle/wrapper/gradle-wrapper.jar differ diff --git a/tools/automatic_query_fixer/gradle/wrapper/gradle-wrapper.properties b/tools/automatic_query_fixer/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..64d92ee58 --- /dev/null +++ b/tools/automatic_query_fixer/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Thu Jun 25 02:01:57 UTC 2020 +distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-all.zip +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStorePath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME diff --git a/tools/automatic_query_fixer/gradlew b/tools/automatic_query_fixer/gradlew new file mode 100755 index 000000000..2fe81a7d9 --- /dev/null +++ b/tools/automatic_query_fixer/gradlew @@ -0,0 +1,183 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/tools/automatic_query_fixer/gradlew.bat b/tools/automatic_query_fixer/gradlew.bat new file mode 100644 index 000000000..9618d8d96 --- /dev/null +++ b/tools/automatic_query_fixer/gradlew.bat @@ -0,0 +1,100 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/tools/automatic_query_fixer/libs/calcite-babel-1.23.0-SNAPSHOT.jar b/tools/automatic_query_fixer/libs/calcite-babel-1.23.0-SNAPSHOT.jar new file mode 100644 index 000000000..cbb42fe82 Binary files /dev/null and b/tools/automatic_query_fixer/libs/calcite-babel-1.23.0-SNAPSHOT.jar differ diff --git a/tools/automatic_query_fixer/libs/calcite-core-1.23.0-SNAPSHOT.jar b/tools/automatic_query_fixer/libs/calcite-core-1.23.0-SNAPSHOT.jar new file mode 100644 index 000000000..2e20949d8 Binary files /dev/null and b/tools/automatic_query_fixer/libs/calcite-core-1.23.0-SNAPSHOT.jar differ diff --git a/tools/automatic_query_fixer/libs/calcite-linq4j-1.23.0-SNAPSHOT.jar b/tools/automatic_query_fixer/libs/calcite-linq4j-1.23.0-SNAPSHOT.jar new file mode 100644 index 000000000..7f093116f Binary files /dev/null and b/tools/automatic_query_fixer/libs/calcite-linq4j-1.23.0-SNAPSHOT.jar differ diff --git a/tools/automatic_query_fixer/settings.gradle b/tools/automatic_query_fixer/settings.gradle new file mode 100644 index 000000000..f2b0bcef6 --- /dev/null +++ b/tools/automatic_query_fixer/settings.gradle @@ -0,0 +1,10 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user manual at https://docs.gradle.org/6.1.1/userguide/multi_project_builds.html + */ + +rootProject.name = 'AutomaticQueryFixer' diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/BigQueryParserFactory.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/BigQueryParserFactory.java new file mode 100644 index 000000000..c7a24e2ba --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/BigQueryParserFactory.java @@ -0,0 +1,74 @@ +package com.google.cloud.bigquery.utils.queryfixer; + +import org.apache.calcite.avatica.util.Casing; +import org.apache.calcite.avatica.util.Quoting; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserImplFactory; +import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; +import org.apache.calcite.sql.validate.SqlConformance; +import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.util.SourceStringReader; + +import java.io.Reader; + +/** + * A factory to generate parsers. The fault generated parser is Babel Parser with BigQuery dialect. + * */ +public class BigQueryParserFactory { + + private final SqlParserImplFactory factory; + private final Quoting quoting = Quoting.BACK_TICK; + private final Casing unquotedCasing = Casing.TO_UPPER; + private final Casing quotedCasing = Casing.UNCHANGED; + private final SqlConformance conformance = SqlConformanceEnum.DEFAULT; + private final SqlParser.Config parserConfig; + + /** + * Default initialization with the Babel Parser factory. + * */ + public BigQueryParserFactory() { + factory = SqlBabelParserImpl.FACTORY; + this.parserConfig = buildConfig(); + } + + /** + * Initialization with customized factory + * @param factory customized factory */ + public BigQueryParserFactory(SqlParserImplFactory factory) { + this.factory = factory; + this.parserConfig = buildConfig(); + } + + /** + * Get the parser parsing the input query. + * @param sql query to parse + * @return a parser loaded with the query + */ + public SqlParser getParser(String sql) { + return getParser(new SourceStringReader(sql)); + } + + /** + * Get the config of the parser factory + * @return parser config + */ + public SqlParser.Config getParserConfig() { + return parserConfig; + } + + protected SqlParser getParser(Reader source) { + return SqlParser.create(source, parserConfig); + } + + private SqlParser.Config buildConfig() { + final SqlParser.ConfigBuilder configBuilder = + SqlParser.configBuilder() + .setParserFactory(factory) + .setQuoting(quoting) + .setUnquotedCasing(unquotedCasing) + .setQuotedCasing(quotedCasing) + .setConformance(conformance); + return configBuilder.build(); + } +} + diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/QueryFixerMain.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/QueryFixerMain.java new file mode 100644 index 000000000..8be498d3e --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/QueryFixerMain.java @@ -0,0 +1,19 @@ +package com.google.cloud.bigquery.utils.queryfixer; + +import com.google.common.flogger.FluentLogger; + +public class QueryFixerMain { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + public static void main(String[] args) { + + if (args.length == 0) { + logger.atInfo().log("not enough arguments"); + return; + } + + String query = args[0]; + // TODO: this should act as a command line, which will be developed after the fixer component is ready. + } +} diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/QueryTokenService.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/QueryTokenService.java new file mode 100644 index 000000000..3352b08a4 --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/QueryTokenService.java @@ -0,0 +1,132 @@ +package com.google.cloud.bigquery.utils.queryfixer; + +import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; +import org.apache.calcite.sql.parser.babel.Token; +import org.apache.calcite.util.SourceStringReader; +import org.apache.commons.lang3.tuple.Pair; + +import com.google.cloud.bigquery.utils.queryfixer.entity.IToken; +import com.google.cloud.bigquery.utils.queryfixer.entity.TokenImpl; + +import lombok.AllArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +/** + * A service provides methods for query and token. It can be used to tokenize queries, find tokens based on + * positions, and modify a query in token-level. + * */ +@AllArgsConstructor +public class QueryTokenService { + + private final static String BackTickQuotingMode = "BTID"; + private final BigQueryParserFactory parserFactory; + + /** + * return a pair of tokens which is closed to a specific position at a query. The left returned token + * is left to the position, and the right returned token is at or right to the position. + * @param sql the input query + * @param line the line of the position. + * @param column the column of the position + * @return a pair of tokens. + */ + public Pair getNearbyTokens(String sql, int line, int column) { + IToken previous = null; + + for (IToken token : getAllTokens(sql)) { + if (token.getBeginLine() >= line && token.getBeginCol() >= column) { + return Pair.of(previous, token); + } + previous = token; + } + + return Pair.of(previous, previous); + } + + /** + * tokenize a query and return all its tokens. + * @param sql the query to be tokenized + * @return a list of tokens of the query + */ + public List getAllTokens(String sql) { + List tokens = new ArrayList<>(); + + // SqlBabelParserImpl has a token manager to tokenize the input query. + SqlBabelParserImpl parserImpl = (SqlBabelParserImpl) parserFactory.getParserConfig() + .parserFactory().getParser(new SourceStringReader(sql)); + parserImpl.switchTo(BackTickQuotingMode); + + Token token; + final int EndKind = 0; + // Token kind means the category: it could be some keywords, identifier, literals, and etc. + // The end of a token stream is end token, whose value is zero. + while ((token = parserImpl.getNextToken()).kind != EndKind) { + tokens.add(new TokenImpl(token)); + } + + return tokens; + } + + /** + * replace a token of a query and return the new query. + * @param sql the query whose token is to be replaced + * @param token the token to be replaced + * @param identifier the identifier the token is placed to. + * @return the replaced query + */ + public String replaceToken(String sql, IToken token, String identifier) { + String[] lines = sql.split("\n"); + validateToken(lines, token); + + // The token's line and column number are 1-index, + // but the array and string index start with 0. + String line = lines[token.getBeginLine() - 1]; + line = replaceStringBetweenIndex(line, token.getBeginCol() - 1, token.getEndCol(), identifier); + lines[token.getBeginLine() - 1] = line; + + return String.join("\n", lines); + } + + /** + * insert an identifier before a token and return the new query. + * @param sql the query to be inserted + * @param token the token to be inserted in front + * @param identifier the inserted identifier + * @return the inserted query + */ + public String insertBeforeToken(String sql, IToken token, String identifier) { + return replaceToken(sql, token, identifier + " " + token.getImage()); + } + + /** + * delete a token from a query and return the new query. + * @param sql the query whose token is to be deleted + * @param token the token to be deleted + * @return the modified query + */ + public String deleteToken(String sql, IToken token) { + return replaceToken(sql, token, ""); + } + + private void validateToken(String[] lines, IToken token) { + if (token.getBeginLine() != token.getEndLine()) { + throw new IllegalArgumentException("Illegal Token"); + } + + if (token.getEndLine() > lines.length) { + throw new IllegalArgumentException("the end line of token exceeds the total length of query"); + } + } + + /** + * replace a substring of a string to a new one. the replacing range is replaced as [startIndex, endIndex), + * i.e. the endIndex is excluded. + */ + private String replaceStringBetweenIndex(String old, int startIndex, int endIndex, String replacingPart) { + StringBuilder builder = new StringBuilder(old); + builder.replace(startIndex, endIndex, replacingPart); + return builder.toString(); + } + +} diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/ChoiceRequest.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/ChoiceRequest.java new file mode 100644 index 000000000..0d108459e --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/ChoiceRequest.java @@ -0,0 +1,28 @@ +package com.google.cloud.bigquery.utils.queryfixer.entity; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +/** + * A request to ask users to choose/confirm an option to fix the query. The request contains error + * position and a list of fix options. + * + * @see package com.google.cloud.bigquery.utils.auto_query_fixer.entity.FixOption + */ +@Data +public class ChoiceRequest { + + Object errPos; + List fixOptions; + + /** create a choiceRequest and initialize its fixOptions List. */ + public ChoiceRequest() { + fixOptions = new ArrayList<>(); + } + + public void addChoice(FixOption fixOption) { + fixOptions.add(fixOption); + } +} diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/FixOption.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/FixOption.java new file mode 100644 index 000000000..67c510d06 --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/FixOption.java @@ -0,0 +1,13 @@ +package com.google.cloud.bigquery.utils.queryfixer.entity; + +import lombok.Value; + +/** + * An option to fix an error of a query. It contains the description on how to fix the error and the + * query after fixing. + */ +@Value +public class FixOption { + String description; + String fixedQuery; +} diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/FixResult.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/FixResult.java new file mode 100644 index 000000000..e5e5d0125 --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/FixResult.java @@ -0,0 +1,22 @@ +package com.google.cloud.bigquery.utils.queryfixer.entity; + + +import lombok.Data; + +/** + * A result on fixing a query. It has three states: SUCCESS, FAIL, REQUEST_RESPONSE. + * SUCCESS means the fixed query is error prone. FAIL means the query contains error(s) and cannot be fixed. + * REQUEST_RESPONSE means the auto fixer needs a response to perform next fixing. + * */ +@Data +public class FixResult { + + private Status status; + private String output; + private String error; + + + private enum Status { + SUCCESS, FAIL, REQUEST_RESPONSE + } +} diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/IToken.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/IToken.java new file mode 100644 index 000000000..75926ce28 --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/IToken.java @@ -0,0 +1,20 @@ +package com.google.cloud.bigquery.utils.queryfixer.entity; + +/** + * A interface for queries' tokens. + * */ +public interface IToken { + + int getKind(); + + String getImage(); + + int getBeginLine(); + + int getBeginCol(); + + int getEndLine(); + + int getEndCol(); + +} diff --git a/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/TokenImpl.java b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/TokenImpl.java new file mode 100644 index 000000000..5006d3ecc --- /dev/null +++ b/tools/automatic_query_fixer/src/main/java/com/google/cloud/bigquery/utils/queryfixer/entity/TokenImpl.java @@ -0,0 +1,52 @@ +package com.google.cloud.bigquery.utils.queryfixer.entity; + +import org.apache.calcite.sql.parser.babel.Token; + +/** + * An implementation of token interface based on the Token class of Babel Calcite Parser. + * */ +public class TokenImpl implements IToken { + + private final Token token; + + public TokenImpl (Token token) { + this.token = token; + } + + @Override public int getKind() { + return token.kind; + } + + @Override public String getImage() { + return token.image; + } + + @Override public int getBeginLine() { + return token.beginLine; + } + + @Override public int getBeginCol() { + return token.beginColumn; + } + + @Override public int getEndLine() { + return token.endLine; + } + + @Override public int getEndCol() { + return token.endColumn; + } + + // babel is a type of parser in Calcite, and this class uses its token for implementation. + public Token getBabelToken() { + return token; + } + + @Override public String toString() { + return String.format("%s [%d:%d]", token.image, token.beginLine, token.beginColumn); + } + + // public static TokenImpl newToken(int ofKind) { +// return new TokenImpl(Token.newToken(ofKind)); +// } +} diff --git a/tools/automatic_query_fixer/src/test/java/com/google/cloud/bigquery/utils/queryfixer/QueryTokenServiceTest.java b/tools/automatic_query_fixer/src/test/java/com/google/cloud/bigquery/utils/queryfixer/QueryTokenServiceTest.java new file mode 100644 index 000000000..dd50d5f1c --- /dev/null +++ b/tools/automatic_query_fixer/src/test/java/com/google/cloud/bigquery/utils/queryfixer/QueryTokenServiceTest.java @@ -0,0 +1,74 @@ +package com.google.cloud.bigquery.utils.queryfixer; + +import org.apache.commons.lang3.tuple.Pair; + +import com.google.cloud.bigquery.utils.queryfixer.entity.IToken; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class QueryTokenServiceTest { + + private QueryTokenService createService() { + return new QueryTokenService(new BigQueryParserFactory()); + } + + @Test + public void convertSqlToTokens() { + QueryTokenService tokenService = createService(); + String sql = "Select col from `d1.t1` Join\n" + + "t2 on d1.t1.id = t2.id\n" + + "where t1.col > 'val'"; + + List tokens = tokenService.getAllTokens(sql); + for (IToken token : tokens) { + System.out.println(token); + } + } + + @Test + public void testNearbyTokens() { + QueryTokenService tokenService = createService(); + String sql = "Select col from t1 Join\n" + + "t2 on t1.id = t2.id\n" + + "where t1.col > 'val'"; + + Pair tokenPair = tokenService.getNearbyTokens(sql, 1, 12); + assertEquals("col", tokenPair.getLeft().getImage()); + assertEquals("from", tokenPair.getRight().getImage()); + + tokenPair = tokenService.getNearbyTokens(sql, 3, 1); + assertEquals("id", tokenPair.getLeft().getImage()); + assertEquals("where", tokenPair.getRight().getImage()); + } + + @Test + public void modifyQuery() { + QueryTokenService tokenService = createService(); + String sql = "Select col from t1 Join\n" + + "t2 on t1.id = t2.id\n" + + "where t1.col > 'val'"; + + String identifier = "Select Distinct"; + + IToken token = tokenService.getNearbyTokens(sql, 1, 1).getRight(); + String modifiedQuery = tokenService.replaceToken(sql, token, identifier); + // modified: Select Distinct col + assertEquals(identifier, modifiedQuery.substring(0, identifier.length())); + + identifier = "Distinct"; + token = tokenService.getNearbyTokens(sql, 1, 8).getRight(); + modifiedQuery = tokenService.insertBeforeToken(sql, token, identifier); + // modified: Select Distinct col + assertEquals(identifier, modifiedQuery.substring(7, 7 + identifier.length())); + + token = tokenService.getNearbyTokens(sql, 1, 8).getRight(); + modifiedQuery = tokenService.deleteToken(sql, token); + // modified: Select from ... + assertEquals("from", modifiedQuery.substring(8, 12)); + } + +}