feat(schema-compiler): Support time series queries in MySQL dialect in Tesseract

.github/workflows/drivers-tests.yml

@@ -285,6 +285,8 @@ jobs:
             use_tesseract_sql_planner: true
           - database: databricks-jdbc
             use_tesseract_sql_planner: true
+          - database: mysql
+            use_tesseract_sql_planner: true
       fail-fast: false
 
     steps:

packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts

@@ -3,20 +3,21 @@ import { getEnv, parseSqlInterval } from '@cubejs-backend/shared';
 import { BaseQuery } from './BaseQuery';
 import { BaseFilter } from './BaseFilter';
 import { UserError } from '../compiler/UserError';
+import { BaseTimeDimension } from './BaseTimeDimension';
 
 const GRANULARITY_TO_INTERVAL = {
-  day: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT00:00:00.000')`,
-  week: (date) => `DATE_FORMAT(DATE_ADD('1900-01-01', INTERVAL TIMESTAMPDIFF(WEEK, '1900-01-01', ${date}) WEEK), '%Y-%m-%dT00:00:00.000')`,
-  hour: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:00:00.000')`,
-  minute: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:%i:00.000')`,
-  second: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:%i:%S.000')`,
-  month: (date) => `DATE_FORMAT(${date}, '%Y-%m-01T00:00:00.000')`,
-  quarter: (date) => `DATE_ADD('1900-01-01', INTERVAL TIMESTAMPDIFF(QUARTER, '1900-01-01', ${date}) QUARTER)`,
-  year: (date) => `DATE_FORMAT(${date}, '%Y-01-01T00:00:00.000')`
+  day: (date: string) => `DATE_FORMAT(${date}, '%Y-%m-%dT00:00:00.000')`,
+  week: (date: string) => `DATE_FORMAT(DATE_ADD('1900-01-01', INTERVAL TIMESTAMPDIFF(WEEK, '1900-01-01', ${date}) WEEK), '%Y-%m-%dT00:00:00.000')`,
+  hour: (date: string) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:00:00.000')`,
+  minute: (date: string) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:%i:00.000')`,
+  second: (date: string) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:%i:%S.000')`,
+  month: (date: string) => `DATE_FORMAT(${date}, '%Y-%m-01T00:00:00.000')`,
+  quarter: (date: string) => `DATE_ADD('1900-01-01', INTERVAL TIMESTAMPDIFF(QUARTER, '1900-01-01', ${date}) QUARTER)`,
+  year: (date: string) => `DATE_FORMAT(${date}, '%Y-01-01T00:00:00.000')`
 };
 
 class MysqlFilter extends BaseFilter {
-  public likeIgnoreCase(column, not, param, type) {
+  public likeIgnoreCase(column: string, not: boolean, param, type: string) {
     const p = (!type || type === 'contains' || type === 'ends') ? '%' : '';
     const s = (!type || type === 'contains' || type === 'starts') ? '%' : '';
     return `${column}${not ? ' NOT' : ''} LIKE CONCAT('${p}', ${this.allocateParam(param)}, '${s}')`;
@@ -125,44 +126,59 @@ export class MysqlQuery extends BaseQuery {
       return `'${intervalParsed.hour}:${intervalParsed.minute}:${intervalParsed.second}' HOUR_SECOND`;
     } else if (intervalParsed.minute && intervalParsed.second && intKeys === 2) {
       return `'${intervalParsed.minute}:${intervalParsed.second}' MINUTE_SECOND`;
+    } else if (intervalParsed.hour && intKeys === 1) {
+      return `${intervalParsed.hour} HOUR`;
+    } else if (intervalParsed.minute && intKeys === 1) {
+      return `${intervalParsed.minute} MINUTE`;
+    } else if (intervalParsed.second && intKeys === 1) {
+      return `${intervalParsed.second} SECOND`;
+    } else if (intervalParsed.millisecond && intKeys === 1) {
+      // MySQL doesn't support MILLISECOND, use MICROSECOND instead (1ms = 1000μs)
+      return `${intervalParsed.millisecond * 1000} MICROSECOND`;
     }
 
-    // No need to support microseconds.
-
     throw new Error(`Cannot transform interval expression "${interval}" to MySQL dialect`);
   }
 
-  public escapeColumnName(name) {
+  public escapeColumnName(name: string): string {
     return `\`${name}\``;
   }
 
-  public seriesSql(timeDimension) {
+  public seriesSql(timeDimension: BaseTimeDimension): string {
     const values = timeDimension.timeSeries().map(
       ([from, to]) => `select '${from}' f, '${to}' t`
     ).join(' UNION ALL ');
     return `SELECT TIMESTAMP(dates.f) date_from, TIMESTAMP(dates.t) date_to FROM (${values}) AS dates`;
   }
 
-  public concatStringsSql(strings) {
+  public concatStringsSql(strings: string[]): string {
     return `CONCAT(${strings.join(', ')})`;
   }
 
-  public unixTimestampSql() {
+  public unixTimestampSql(): string {
     return 'UNIX_TIMESTAMP()';
   }
 
-  public wrapSegmentForDimensionSelect(sql) {
+  public wrapSegmentForDimensionSelect(sql: string): string {
     return `IF(${sql}, 1, 0)`;
   }
 
-  public preAggregationTableName(cube, preAggregationName, skipSchema) {
+  public preAggregationTableName(cube: string, preAggregationName: string, skipSchema: boolean): string {
     const name = super.preAggregationTableName(cube, preAggregationName, skipSchema);
     if (name.length > 64) {
       throw new UserError(`MySQL can not work with table names that longer than 64 symbols. Consider using the 'sqlAlias' attribute in your cube and in your pre-aggregation definition for ${name}.`);
     }
     return name;
   }
 
+  public supportGeneratedSeriesForCustomTd(): boolean {
+    return true;
+  }
+
+  public intervalString(interval: string): string {
+    return this.formatInterval(interval);
+  }
+
   public sqlTemplates() {
     const templates = super.sqlTemplates();
     // PERCENTILE_CONT works but requires PARTITION BY
@@ -172,11 +188,51 @@ export class MysqlQuery extends BaseQuery {
     // NOTE: this template contains a comma; two order expressions are being generated
     templates.expressions.sort = '{{ expr }} IS NULL {% if nulls_first %}DESC{% else %}ASC{% endif %}, {{ expr }} {% if asc %}ASC{% else %}DESC{% endif %}';
     delete templates.expressions.ilike;
-    templates.types.string = 'VARCHAR';
+    templates.types.string = 'CHAR';
     templates.types.boolean = 'TINYINT';
     templates.types.timestamp = 'DATETIME';
     delete templates.types.interval;
     templates.types.binary = 'BLOB';
+
+    templates.expressions.concat_strings = 'CONCAT({{ strings | join(\',\' ) }})';
+
+    templates.filters.like_pattern = 'CONCAT({% if start_wild %}\'%\'{% else %}\'\'{% endif %}, LOWER({{ value }}), {% if end_wild %}\'%\'{% else %}\'\'{% endif %})';
+    templates.tesseract.ilike = 'LOWER({{ expr }}) {% if negated %}NOT {% endif %}LIKE {{ pattern }}';
+
+    templates.statements.time_series_select = 'SELECT TIMESTAMP(dates.f) date_from, TIMESTAMP(dates.t) date_to \n' +
+      'FROM (\n' +
+      '{% for time_item in seria  %}' +
+      '    select \'{{ time_item[0] }}\' f, \'{{ time_item[1] }}\' t \n' +
+      '{% if not loop.last %} UNION ALL\n{% endif %}' +
+      '{% endfor %}' +
+      ') AS dates';
+
+    templates.statements.generated_time_series_select =
+      'WITH RECURSIVE date_series AS (\n' +
+      '  SELECT TIMESTAMP({{ start }}) AS date_from\n' +
+      '  UNION ALL\n' +
+      '  SELECT DATE_ADD(date_from, INTERVAL {{ granularity }})\n' +
+      '  FROM date_series\n' +
+      '  WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= TIMESTAMP({{ end }})\n' +
+      ')\n' +
+      'SELECT CAST(date_from AS DATETIME) AS date_from,\n' +
+      '       CAST(DATE_SUB(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME) AS date_to\n' +
+      'FROM date_series';
+
+    templates.statements.generated_time_series_with_cte_range_source =
+      'WITH RECURSIVE date_series AS (\n' +
+      '  SELECT {{ range_source }}.{{ min_name }} AS date_from,\n' +
+      '         {{ range_source }}.{{ max_name }} AS max_date\n' +
+      '  FROM {{ range_source }}\n' +
+      '  UNION ALL\n' +
+      '  SELECT DATE_ADD(date_from, INTERVAL {{ granularity }}), max_date\n' +
+      '  FROM date_series\n' +
+      '  WHERE DATE_ADD(date_from, INTERVAL {{ granularity }}) <= max_date\n' +
+      ')\n' +
+      'SELECT CAST(date_from AS DATETIME) AS date_from,\n' +
+      '       CAST(DATE_SUB(DATE_ADD(date_from, INTERVAL {{ granularity }}), INTERVAL 1000 MICROSECOND) AS DATETIME) AS date_to\n' +
+      'FROM date_series';
+
     return templates;
   }
 }

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -174,11 +174,50 @@
     "SQL API: Extended nested Rollup over asterisk",
     "SQL API: SQL push down push to cube quoted alias",
     "SQL API: Date/time comparison with date_trunc with SQL push down",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
+  ],
+  "tesseractSkip": [
+    "querying custom granularities ECommerce: count by three_months_by_march + no dimension",
+    "querying custom granularities ECommerce: count by three_months_by_march + dimension",
+    "querying BigECommerce: rolling window YTD (month + week)",
+    "querying BigECommerce: rolling window YTD (month + week + no gran)",
+
 
     "---------------------------------------",
-    "Error during rewrite: Can't detect Cube query and it may be not supported yet.",
+    "SKIPPED SQL API (Need work)",
     "---------------------------------------",
+    "SQL API: reuse params",
+    "SQL API: Nested Rollup",
+    "SQL API: Nested Rollup with aliases",
     "SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
-    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
+    "SQL API: Rolling Window YTD (year + month + day + date_trunc IN)",
+    "SQL API: SQL push down push to cube quoted alias",
+    "SQL API: Date/time comparison with date_trunc with SQL push down",
+
+    "---- Different results comparing to baseQuery version. Need to investigate ----",
+    "SQL API: Timeshift measure from cube",
+    "querying ECommerce: dimensions",
+    "querying ECommerce: dimensions + order",
+    "querying ECommerce: dimensions + limit",
+    "querying ECommerce: dimensions + total",
+    "querying ECommerce: dimensions + order + limit + total",
+    "querying ECommerce: dimensions + order + total + offset",
+    "querying ECommerce: dimensions + order + limit + total + offset",
+    "filtering ECommerce: contains dimensions, first",
+    "filtering ECommerce: contains dimensions, second",
+    "filtering ECommerce: startsWith + dimensions, first",
+    "filtering ECommerce: startsWith + dimensions, second",
+    "filtering ECommerce: endsWith + dimensions, first",
+    "filtering ECommerce: endsWith + dimensions, second",
+    "querying BigECommerce: rolling window YTD without date range",
+    "querying BigECommerce: rolling window YTD (month + week + day + no gran)",
+    "querying BigECommerce: rolling window YTD (month + week + day)",
+    "querying BigECommerce: rolling window YTD (month)",
+    "querying BigECommerce: rolling window by 2 month without date range",
+    "querying BigECommerce: rolling window by 2 month",
+    "querying BigECommerce: rolling window by 2 week",
+    "querying BigECommerce: rolling window by 2 day without date range",
+    "querying BigECommerce: rolling window by 2 day"
   ]
 }