clickhouse-cpp 2.1.0; DateTime64 support

.Rbuildignore

@@ -16,3 +16,4 @@
 ^LICENSE$
 ^tests/testthat/test-dbplyr-ch.R$
 .ipynb_checkpoints
+^src/vendor/clickhouse-cpp/build

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: RClickhouse
 Type: Package
 Title: 'Yandex Clickhouse' Interface for R with Basic 'dplyr' Support
-Version: 0.6.3
+Version: 0.6.4
 Encoding: UTF-8
 Authors@R: c(
     person("Christian", "Hotz-Behofsits", email = "christian.hotz-behofsits@wu.ac.at", role = c("aut", "cre")),
@@ -22,9 +22,11 @@ Authors@R: c(
 Description: 'Yandex Clickhouse' (<https://clickhouse.com/>) is a high-performance relational column-store database to enable
     big data exploration and 'analytics' scaling to petabytes of data. Methods are
     provided that enable working with 'Yandex Clickhouse' databases via
-    'DBI' methods and using 'dplyr'/'dbplyr' idioms.
+    'DBI' methods and using 'dplyr'/'dbplyr' idioms. 
+    NOTE(mmitkevich): https://github.com/ClickHouse/clickhouse-cpp/commit/87fc1186063b82b8ddf37b1c70c485bdd7504f8d
+          https://github.com/mmitkevich/RClickhouse
 License: GPL-2
-SystemRequirements: C++11
+SystemRequirements: C++17
 Imports:
     dplyr (>= 0.7.0),
     dbplyr (>= 1.0.0),

Makefile

@@ -0,0 +1,33 @@
+# Makefile for generating R packages.
+# 2011 Andrew Redd
+#
+# Assumes Makefile is in a folder where package contents are in a subfolder pkg.
+# Roxygen uses the roxygen2 package, and will run automatically on check and all.
+
+PKG_VERSION=$(shell grep -i ^version DESCRIPTION | cut -d : -d \  -f 2)
+PKG_NAME=$(shell grep -i ^package DESCRIPTION | cut -d : -d \  -f 2)
+
+R_FILES := $(wildcard R/*.R)
+SRC_FILES := $(wildcard R/*.R)
+PKG_FILES := DESCRIPTION NAMESPACE $(R_FILES) $(SRC_FILES)
+
+.PHONY: tarball install check clean roxygen
+
+tarball: $(PKG_NAME)_$(PKG_VERSION).tar.gz
+$(PKG_NAME)_$(PKG_VERSION).tar.gz: $(PKG_FILES)
+	R CMD build .
+
+all: clean check install
+
+check: $(PKG_NAME)_$(PKG_VERSION).tar.gz roxygen
+	R CMD check $(PKG_NAME)_$(PKG_VERSION).tar.gz
+
+install: $(PKG_NAME)_$(PKG_VERSION).tar.gz
+	R CMD INSTALL $(PKG_NAME)_$(PKG_VERSION).tar.gz
+
+roxygen:
+	Rscript -e "library(roxygen2);roxygenize('.')"
+
+clean:
+	-rm -f $(PKG_NAME)_*.tar.gz
+	-rm -r -f $(PKG_NAME).Rcheck

NAMESPACE

@@ -56,4 +56,5 @@ importFrom(dplyr,db_explain)
 importFrom(dplyr,sql_escape_ident)
 importFrom(dplyr,sql_escape_string)
 importFrom(dplyr,sql_translate_env)
+import(openssl)
 useDynLib(RClickhouse, .registration=TRUE)

src/Makevars

@@ -1 +1 @@
-include Makevars.common
+include Makevars.common

src/Makevars.common

@@ -1,35 +1,38 @@
-PKG_CPPFLAGS = $(SYS_FLAGS) -I. -I../inst/include -I./vendor/clickhouse-cpp -I./vendor/clickhouse-cpp/contrib -I./vendor/clickhouse-cpp/contrib/bigerint
-
-CXX_STD = CXX11
+CXX_STD = CXX17
+PKG_CPPFLAGS = $(SYS_FLAGS) -I. -I../inst/include -I./vendor/clickhouse-cpp -I./vendor/clickhouse-cpp/contrib
+# -I./vendor/clickhouse-cpp/contrib/bigerint
 
 OBJ_FILES = \
-vendor/clickhouse-cpp/clickhouse/columns/string.o \
-vendor/clickhouse-cpp/clickhouse/columns/date.o \
-vendor/clickhouse-cpp/clickhouse/columns/numeric.o \
-vendor/clickhouse-cpp/clickhouse/columns/decimal.o \
-vendor/clickhouse-cpp/clickhouse/columns/tuple.o \
-vendor/clickhouse-cpp/clickhouse/columns/array.o \
-vendor/clickhouse-cpp/clickhouse/columns/factory.o \
-vendor/clickhouse-cpp/clickhouse/columns/nullable.o \
-vendor/clickhouse-cpp/clickhouse/columns/enum.o \
-vendor/clickhouse-cpp/clickhouse/columns/uuid.o \
-vendor/clickhouse-cpp/clickhouse/columns/ip4.o \
-vendor/clickhouse-cpp/clickhouse/columns/ip6.o \
-vendor/clickhouse-cpp/clickhouse/query.o \
-vendor/clickhouse-cpp/clickhouse/base/platform.o \
 vendor/clickhouse-cpp/clickhouse/base/socket.o \
+vendor/clickhouse-cpp/clickhouse/base/platform.o \
+vendor/clickhouse-cpp/clickhouse/base/sslsocket.o \
 vendor/clickhouse-cpp/clickhouse/base/input.o \
 vendor/clickhouse-cpp/clickhouse/base/output.o \
-vendor/clickhouse-cpp/clickhouse/base/coded.o \
 vendor/clickhouse-cpp/clickhouse/base/compressed.o \
-vendor/clickhouse-cpp/clickhouse/client.o \
+vendor/clickhouse-cpp/clickhouse/base/wire_format.o \
+vendor/clickhouse-cpp/clickhouse/block.o \
+vendor/clickhouse-cpp/clickhouse/query.o \
 vendor/clickhouse-cpp/clickhouse/types/types.o \
 vendor/clickhouse-cpp/clickhouse/types/type_parser.o \
-vendor/clickhouse-cpp/clickhouse/block.o \
-vendor/clickhouse-cpp/contrib/cityhash/city.o \
+vendor/clickhouse-cpp/clickhouse/client.o \
+vendor/clickhouse-cpp/clickhouse/columns/ip6.o \
+vendor/clickhouse-cpp/clickhouse/columns/ip4.o \
+vendor/clickhouse-cpp/clickhouse/columns/lowcardinality.o \
+vendor/clickhouse-cpp/clickhouse/columns/numeric.o \
+vendor/clickhouse-cpp/clickhouse/columns/decimal.o \
+vendor/clickhouse-cpp/clickhouse/columns/array.o \
+vendor/clickhouse-cpp/clickhouse/columns/factory.o \
+vendor/clickhouse-cpp/clickhouse/columns/date.o \
+vendor/clickhouse-cpp/clickhouse/columns/enum.o \
+vendor/clickhouse-cpp/clickhouse/columns/tuple.o \
+vendor/clickhouse-cpp/clickhouse/columns/itemview.o \
+vendor/clickhouse-cpp/clickhouse/columns/string.o \
+vendor/clickhouse-cpp/clickhouse/columns/nullable.o \
+vendor/clickhouse-cpp/clickhouse/columns/uuid.o \
+vendor/clickhouse-cpp/contrib/lz4/lz4hc.o \
 vendor/clickhouse-cpp/contrib/lz4/lz4.o \
-vendor/clickhouse-cpp/contrib/lz4/lz4hc.o
+vendor/clickhouse-cpp/contrib/cityhash/city.o
 
-PKG_LIBS = $(OBJ_FILES) -lpthread $(SYS_LIBS)
+PKG_LIBS = $(OBJ_FILES) -lpthread -lssl $(SYS_LIBS)
 
 $(SHLIB): $(OBJ_FILES)

src/RcppExports.cpp

@@ -8,6 +8,11 @@
 
 using namespace Rcpp;
 
+#ifdef RCPP_USE_GLOBAL_ROSTREAM
+Rcpp::Rostream<true>&  Rcpp::Rcout = Rcpp::Rcpp_cout_get();
+Rcpp::Rostream<false>& Rcpp::Rcerr = Rcpp::Rcpp_cerr_get();
+#endif
+
 // fetch
 DataFrame fetch(XPtr<Result> res, ssize_t n);
 static SEXP _RClickhouse_fetch_try(SEXP resSEXP, SEXP nSEXP) {

src/result.cpp

@@ -1,7 +1,8 @@
 #include <stdexcept>
+#include <type_traits>
+#include "clickhouse/columns/string.h"
 #include "result.h"
-
-
+
 // helper function which emits an R warning without causing a longjmp
 // see https://stackoverflow.com/questions/24557711/how-to-generate-an-r-warning-safely-in-rcpp
 void warn(std::string text) {
@@ -62,6 +63,37 @@ void convertEntries(std::shared_ptr<const CT> in, NullCol nullCol, RT &out,
 }
 
 
+// workaround lack of Rcpp::StringVector x; x[i] = std::string_view();
+template<>
+void convertEntries<ch::ColumnString, Rcpp::StringVector>(std::shared_ptr<const ch::ColumnString> in, NullCol nullCol, Rcpp::StringVector &out,
+                    size_t offset, size_t start, size_t end)
+{
+  for(size_t j = start; j < end; j++) {
+    // can't use the ternary operator here, since that would require explicit
+    // conversion from the Clickhouse storage type (which is far messier)
+    if(nullCol && nullCol->IsNull(j)) {
+      out[offset+j-start] = Rcpp::StringVector::get_na();
+    } else {
+      out[offset+j-start] = std::string(in->At(j));
+    }
+  }
+}
+
+template<>
+void convertEntries<ch::ColumnFixedString, Rcpp::StringVector>(std::shared_ptr<const ch::ColumnFixedString> in, NullCol nullCol, Rcpp::StringVector &out,
+                    size_t offset, size_t start, size_t end)
+{
+  for(size_t j = start; j < end; j++) {
+    // can't use the ternary operator here, since that would require explicit
+    // conversion from the Clickhouse storage type (which is far messier)
+    if(nullCol && nullCol->IsNull(j)) {
+      out[offset+j-start] = Rcpp::StringVector::get_na();
+    } else {
+      out[offset+j-start] = std::string(in->At(j));
+    }
+  }
+}
+
 template<>
 void convertEntries<ch::ColumnInt64, Rcpp::StringVector>(std::shared_ptr<const ch::ColumnInt64> in, NullCol nullCol, Rcpp::StringVector &out,
                     size_t offset, size_t start, size_t end) {
@@ -104,7 +136,18 @@ void convertEntries<ch::ColumnDate, Rcpp::DateVector>(std::shared_ptr<const ch::
     }
   }
 }
-
+template<>
+void convertEntries<ch::ColumnDateTime64, Rcpp::DatetimeVector>(std::shared_ptr<const ch::ColumnDateTime64> in,
+    NullCol nullCol, Rcpp::DatetimeVector &out, size_t offset, size_t start, size_t end) {
+  double factor = std::pow(10.,in->GetPrecision());
+  for(size_t j = start; j < end; j++) {
+    if(nullCol && nullCol->IsNull(j)) {
+      out[offset+j-start] = Rcpp::DateVector::get_na();
+    } else {
+      out[offset+j-start] = static_cast<double>(in->At(j)/factor);
+    }
+  }
+}
 std::string formatUUID(const ch::UInt128 &v) {
   const size_t bufsize = 128/4 + 4 + 1;  // 128 bit in hexadecimal + 4 dashes + null terminator
   char buf[bufsize];
@@ -292,6 +335,9 @@ std::unique_ptr<Converter> Result::buildConverter(std::string name, ch::TypeRef
       return std::unique_ptr<ScalarConverter<ch::ColumnDateTime, Rcpp::DatetimeVector>>(new ScalarConverter<ch::ColumnDateTime, Rcpp::DatetimeVector>);
     case TC::Date:
       return std::unique_ptr<ScalarConverter<ch::ColumnDate, Rcpp::DateVector>>(new ScalarConverter<ch::ColumnDate, Rcpp::DateVector>);
+    case TC::DateTime64:
+      return std::unique_ptr<ScalarConverter<ch::ColumnDateTime64, Rcpp::DatetimeVector>>(new ScalarConverter<ch::ColumnDateTime64, Rcpp::DatetimeVector>);
+
     case TC::Nullable:
       {
         // downcast to NullableType to access GetNestedType member

src/vendor/clickhouse-cpp/.gitignore

@@ -274,3 +274,7 @@ BUCKAROO_DEPS
 
 # Visual Studio Code
 /.vscode/
+
+# Vim
+*.swp
+*.swo

src/vendor/clickhouse-cpp/.travis.yml

@@ -35,20 +35,28 @@ matrix:
         - MATRIX_EVAL="CC=clang-6.0 && CXX=clang++-6.0"
 
     - os: osx
-      osx_image: xcode8.2
+      osx_image: xcode9.4
       compiler: clang
 
-before_install:
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo 'deb http://repo.yandex.ru/clickhouse/deb/stable main/' | sudo tee /etc/apt/sources.list.d/clickhouse.list ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get update -q && sudo apt-get install -q -y --allow-unauthenticated clickhouse-server-common ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo service clickhouse-server start ; fi
+before_install: |
+  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    export CH_SERVER_VERSION="21.3.17.2"
+    sudo apt-get install apt-transport-https ca-certificates dirmngr
+    sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4
+    echo "deb https://repo.clickhouse.com/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list
+    sudo apt-get update
+    sudo apt-get install -y \
+        clickhouse-server=${CH_SERVER_VERSION} \
+        clickhouse-client=${CH_SERVER_VERSION} \
+        clickhouse-common-static=${CH_SERVER_VERSION}
+    sudo service clickhouse-server start
+  fi
 
 # Build steps
-script:
-  - eval "${MATRIX_EVAL}"
-  - mkdir build
-  - cd build
-  - cmake .. -DBUILD_TESTS=ON && make
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./ut/clickhouse-cpp-ut ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./ut/clickhouse-cpp-ut --gtest_filter='-Client/*' ; fi
+script: |
+  eval "${MATRIX_EVAL}"
+  mkdir build
+  cd build
+  cmake .. -DBUILD_TESTS=ON && make
+  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./ut/clickhouse-cpp-ut ; fi
+  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./ut/clickhouse-cpp-ut --gtest_filter=-"Client/*:*Performance*" ; fi

src/vendor/clickhouse-cpp/CMakeLists.txt

@@ -1,46 +1,50 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 3.0.2)
+CMAKE_MINIMUM_REQUIRED (VERSION 3.0.2)
 
-INCLUDE (cmake/cpp11.cmake)
+INCLUDE (cmake/cpp17.cmake)
 INCLUDE (cmake/subdirs.cmake)
+INCLUDE (cmake/openssl.cmake)
 
-OPTION(BUILD_BENCHMARK "Build benchmark" OFF)
-OPTION(BUILD_TESTS "Build tests" OFF)
+OPTION (BUILD_BENCHMARK "Build benchmark" OFF)
+OPTION (BUILD_TESTS "Build tests" OFF)
+OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF)
 
 PROJECT (CLICKHOUSE-CLIENT)
 
-    USE_CXX11()
+    USE_CXX17 ()
+    USE_OPENSSL ()
 
-	IF ("${CMAKE_BUILD_TYPE}" STREQUAL "")
-		set(CMAKE_BUILD_TYPE "Debug")
-	ENDIF()
+    IF (NOT CMAKE_BUILD_TYPE)
+        SET (CMAKE_BUILD_TYPE "RelWithDebInfo")
+    ENDIF ()
 
     IF (UNIX)
-        IF (APPLE)
-            SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall -Wextra -Werror")
-        ELSE ()
-            SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -pthread -Wall -Wextra -Werror")
+        IF (NOT APPLE)
+            SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
         ENDIF ()
         SET (CMAKE_EXE_LINKER_FLAGS, "${CMAKE_EXE_LINKER_FLAGS} -lpthread")
+        # -Wpedantic makes int128 support somewhat harder and less performant (by not allowing builtin __int128)
+        SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wpedantic-")
     ENDIF ()
 
-    INCLUDE_DIRECTORIES(.)
-    INCLUDE_DIRECTORIES(contrib)
+    INCLUDE_DIRECTORIES (.)
+    INCLUDE_DIRECTORIES (contrib)
 
     SUBDIRS (
         clickhouse
+        contrib/absl
         contrib/cityhash
         contrib/lz4
-        contrib/bigint
     )
 
     IF (BUILD_BENCHMARK)
-        SUBDIRS(bench)
+        SUBDIRS (bench)
     ENDIF (BUILD_BENCHMARK)
 
     IF (BUILD_TESTS)
-        SUBDIRS(
+        INCLUDE_DIRECTORIES (contrib/gtest/include contrib/gtest)
+        SUBDIRS (
             contrib/gtest
             tests/simple
             ut
         )
-	ENDIF (BUILD_TESTS)
+    ENDIF (BUILD_TESTS)

src/vendor/clickhouse-cpp/LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018-2019 Yandex LLC
+Copyright 2018-2020 Yandex LLC
 Copyright 2017 Pavel Artemkin
 
                                  Apache License
@@ -189,7 +189,7 @@ Copyright 2017 Pavel Artemkin
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2018-2019 Yandex LLC
+   Copyright 2018-2020 Yandex LLC
    Copyright 2017 Pavel Artemkin
 
 

src/vendor/clickhouse-cpp/README.md

@@ -1,22 +1,26 @@
-ClickHouse C++ client [![Build Status](https://travis-ci.org/ClickHouse/clickhouse-cpp.svg?branch=master)](https://travis-ci.org/ClickHouse/clickhouse-cpp)
+ClickHouse C++ client [![Linux](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/linux.yml/badge.svg)](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/linux.yml) [![macOS](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/macos.yml/badge.svg)](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/macos.yml) [![Windows MSVC](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/windows_msvc.yml/badge.svg)](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/windows_msvc.yml) [![Windows mingw](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/windows_mingw.yml/badge.svg)](https://github.com/ClickHouse/clickhouse-cpp/actions/workflows/windows_mingw.yml)
 =====
 
-C++ client for [Yandex ClickHouse](https://clickhouse.yandex/)
+C++ client for [ClickHouse](https://clickhouse.com/).
 
 ## Supported data types
 
 * Array(T)
 * Date
-* DateTime
+* DateTime, DateTime64
+* DateTime([timezone]), DateTime64(N, [timezone])
 * Decimal32, Decimal64, Decimal128
 * Enum8, Enum16
 * FixedString(N)
 * Float32, Float64
 * IPv4, IPv6
 * Nullable(T)
 * String
+* LowCardinality(String) or LowCardinality(FixedString(N))
 * Tuple
 * UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64
+* Int128
+* UUID
 
 ## Building
 
@@ -71,3 +75,7 @@ client.Select("SELECT id, name FROM test.numbers", [] (const Block& block)
 /// Delete table.
 client.Execute("DROP TABLE test.numbers");
 ```
+Please note that `Client` instance is NOT thread-safe. I.e. you must create a separate `Client` for each thread or utilize some synchronization techniques.
+
+
+source code taken from: https://github.com/ClickHouse/clickhouse-cpp/commit/87fc1186063b82b8ddf37b1c70c485bdd7504f8d