/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.integration.spark.testsuite.primitiveTypes

import java.io.File

import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll

import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties

class TestAdaptiveEncodingForPrimitiveTypes extends QueryTest with BeforeAndAfterAll {

  val rootPath = new File(this.getClass.getResource("/").getPath
                          + "../../../..").getCanonicalPath

  private val vectorReader = CarbonProperties.getInstance()
    .getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER)

  CarbonProperties.getInstance()
    .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "2,2")

  override def beforeAll: Unit = {
    dropTables
    sql(
      "CREATE TABLE uniqdata_Compare (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, " +
      "DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) STORED AS carbondata")

    sql(s"LOAD DATA INPATH '${resourcesPath + "/data_with_all_types.csv"}' into table" +
        " uniqdata_Compare OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")

    sql(
      "CREATE TABLE uniqdata_Compaction (CUST_ID int,CUST_NAME String," +
      "ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint," +
      "BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10)," +
      "Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED AS carbondata")

    sql(s"LOAD DATA INPATH '${resourcesPath + "/data_with_all_types.csv"}' into table" +
        " uniqdata_Compaction OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")

    sql(s"LOAD DATA INPATH '${resourcesPath + "/data_with_all_types.csv"}' into table" +
        " uniqdata_Compaction OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")

    sql("alter table uniqdata_Compaction compact 'minor'")

    sql("create table negativeTable_Compare (" +
        "intColumn int,stringColumn string,shortColumn short) STORED AS carbondata")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' into table " +
        "negativeTable_Compare options('FILEHEADER'='intColumn,stringColumn,shortColumn')")

    sql("create table negativeTable_Compaction (" +
        "intColumn int,stringColumn string,shortColumn short) STORED AS carbondata")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' into table " +
        s"negativeTable_Compaction options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' into table " +
        s"negativeTable_Compaction options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    sql("alter table negativeTable_Compaction compact 'minor'")
  }

  test("test adaptive encoding on all possible data types by dictionary exclude") {
    sql("drop table if exists uniqdata")
    sql(
      "CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
      "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) STORED AS carbondata")

    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compare"))
  }

  test("test adaptive encoding on all possible data types by SORT_COLUMNS") {
    sql("drop table if exists uniqdata")
    sql(
      "CREATE TABLE uniqdata (CUST_ID short,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
      "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) STORED AS carbondata TBLPROPERTIES" +
      "('SORT_COLUMNS'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
      "BIGINT_COLUMN2,INTEGER_COLUMN1')")

    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compare"))
  }

  test("test filter queries on the adaptive encoded column") {
    // scalastyle:off lineLength
    sql("drop table if exists negativeTable")
    sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) STORED AS carbondata TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))
    checkAnswer(sql("select * from negativeTable where intColumn<0"), sql("select * from negativeTable_Compare where intColumn<0"))
    checkAnswer(sql("select * from negativeTable where intColumn<=0"), sql("select * from negativeTable_Compare where intColumn<=0"))
    checkAnswer(sql("select * from negativeTable where intColumn>0"), sql("select * from negativeTable_Compare where intColumn>0"))
    checkAnswer(sql("select * from negativeTable where intColumn>=0"), sql("select * from negativeTable_Compare where intColumn>=0"))
    checkAnswer(sql("select * from negativeTable where intColumn >= -20000"), sql("select * from negativeTable_Compare where intColumn >= -20000"))
    checkAnswer(sql("select * from negativeTable where intColumn <= 10000"), sql("select * from negativeTable_Compare where intColumn <= 10000"))
    checkAnswer(sql("select * from negativeTable where intColumn between -20000 and 10000"), sql("select * from negativeTable_Compare where intColumn between -20000 and 10000"))
    checkAnswer(sql("select * from negativeTable where intColumn is null"), sql("select * from negativeTable_Compare where intColumn is null"))
    checkAnswer(sql("select * from negativeTable where intColumn is not null"), sql("select * from negativeTable_Compare where intColumn is not null"))
    checkAnswer(sql("select * from negativeTable where shortColumn<0"), sql("select * from negativeTable_Compare where shortColumn<0"))
    checkAnswer(sql("select * from negativeTable where shortColumn<=0"), sql("select * from negativeTable_Compare where shortColumn<=0"))
    checkAnswer(sql("select * from negativeTable where shortColumn>0"), sql("select * from negativeTable_Compare where shortColumn>0"))
    checkAnswer(sql("select * from negativeTable where shortColumn>=0"), sql("select * from negativeTable_Compare where shortColumn>=0"))
    checkAnswer(sql("select * from negativeTable where shortColumn between -200 and 200"), sql("select * from negativeTable_Compare where shortColumn between -200 and 200"))
    checkAnswer(sql("select * from negativeTable where shortColumn >= -200"), sql("select * from negativeTable_Compare where shortColumn >= -200"))
    checkAnswer(sql("select * from negativeTable where shortColumn <= 100"), sql("select * from negativeTable_Compare where shortColumn <= 100"))
    checkAnswer(sql("select * from negativeTable where shortColumn is null"), sql("select * from negativeTable_Compare where shortColumn is null"))
    checkAnswer(sql("select * from negativeTable where shortColumn is not null"), sql("select * from negativeTable_Compare where shortColumn is not null"))
    sql("drop table if exists negativeTable")
    // scalastyle:on lineLength
  }

  test("test adaptive encoding on all possible data types by sort_columns - vector enable") {
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER, "true")
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
    sql(
      "CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
      "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) STORED AS carbondata TBLPROPERTIES" +
      "('SORT_COLUMNS'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
      "BIGINT_COLUMN2,INTEGER_COLUMN1')")

    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compare"))
    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    sql("alter table uniqdata compact 'minor'")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compaction"))

    // negative data compaction test
    sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) " +
        "STORED AS carbondata TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    sql("alter table negativeTable compact 'minor'")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compaction"))

    CarbonProperties.getInstance()
      .addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER, vectorReader)
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
  }

  test("test adaptive encoding on all possible data types by sort_columns - vector disable") {
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER, "false")
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
    sql(
      "CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
      "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) STORED AS carbondata TBLPROPERTIES" +
      "('SORT_COLUMNS'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
      "BIGINT_COLUMN2,INTEGER_COLUMN1')")

    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compare"))
    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    sql("alter table uniqdata compact 'minor'")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compaction"))

    // negative data compaction test
    sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) " +
        "STORED AS carbondata TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    sql("alter table negativeTable compact 'minor'")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compaction"))

    CarbonProperties.getInstance()
      .addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER, vectorReader)
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
  }

  test("test adaptive encoding on all possible data types by sort_columns - enable unsafe") {
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "true")
        .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, "true")
        .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION, "true")
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
    sql(
      "CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
      "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) STORED AS carbondata TBLPROPERTIES" +
      "('SORT_COLUMNS'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
      "BIGINT_COLUMN2,INTEGER_COLUMN1')")

    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compare"))

    // negative data compaction test
    sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) " +
        "STORED AS carbondata TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))

    restoreDefaultProperty
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
  }

  test("test adaptive encoding on all possible data types by sort_columns - disable unsafe") {
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "false")
      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE, "false")
      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION, "false")
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
    sql(
      "CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
      "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) STORED AS carbondata TBLPROPERTIES" +
      "('SORT_COLUMNS'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
      "BIGINT_COLUMN2,INTEGER_COLUMN1')")

    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    checkAnswer(sql("select * from uniqdata"), sql("select * from uniqdata_Compare"))

    // negative data compaction test
    sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) " +
        "STORED AS carbondata TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))

    restoreDefaultProperty
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
  }

  test("test adaptive encoding on partition table") {
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
    sql("CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
      "timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
      "double,INTEGER_COLUMN1 int) PARTITIONED BY(DOJ timestamp) STORED AS carbondata " +
      "TBLPROPERTIES('SORT_COLUMNS'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ," +
      "BIGINT_COLUMN1,BIGINT_COLUMN2,INTEGER_COLUMN1')")

    sql(s"LOAD DATA INPATH '${ resourcesPath + "/data_with_all_types.csv" }' into table" +
        " uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='\"'," +
        "'BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION," +
        "DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2," +
        "Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')")
    checkAnswer(
      sql("select CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
          "BIGINT_COLUMN2,INTEGER_COLUMN1 from uniqdata"),
      sql("select CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1," +
          "BIGINT_COLUMN2,INTEGER_COLUMN1 from uniqdata_Compare"))

    // negative data compaction test
    sql("create table negativeTable (intColumn int,stringColumn string) " +
        "PARTITIONED BY(shortColumn short) STORED AS carbondata " +
        "TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select intColumn,stringColumn,shortColumn from negativeTable"),
      sql("select intColumn,stringColumn,shortColumn from negativeTable_Compare"))
    sql("drop table if exists uniqdata")
    sql("drop table if exists negativeTable")
  }

  test("test IUD on the adptive encoded column") {
    sql("drop table if exists negativeTable")
    sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) " +
        "STORED AS carbondata TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))
    sql("insert into negativeTable select 0,null,-10")
    sql("insert into negativeTable select null,'inserted',20")
    checkAnswer(sql("select * from negativeTable where intColumn=0"),
      Seq(Row(0, "ddd", 0), Row(0, null, -10)))
    checkAnswer(sql("select * from negativeTable where intColumn is null"),
      Seq(Row(null, "null", null), Row(null, "inserted", 20)))
    sql("update negativeTable set (intColumn) = (5) where intColumn=0").collect()
    checkAnswer(sql("select * from negativeTable where intColumn=5"),
      Seq(Row(5, "ddd", 0), Row(5, null, -10)))
    sql("delete from negativeTable where intColumn=5").collect()
    checkAnswer(sql("select * from negativeTable"),
      Seq(Row(null, "null", null),
        Row(null, "inserted", 20),
        Row(-30000, "aaa", -300),
        Row(-20000, "bbb", -200),
        Row(-10000, "ccc", -100),
        Row(10000, "eee", 100),
        Row(70000, "ggg", 700)))
    sql("drop table if exists negativeTable")
  }

  test("test filter queries on adaptive encoded column with complex column in the schema") {
    sql("drop table if exists complexTable")
    sql("CREATE TABLE complexTable( id LONG,name STRING,salary FLOAT,file " +
        "struct<school:array<string>, age:int>) STORED AS carbondata " +
        "TBLPROPERTIES('sort_columns'='id,name')")
    sql(s"LOAD DATA INPATH '$rootPath/examples/spark/src/main/resources/streamSample.csv' " +
        s"INTO TABLE complexTable OPTIONS('HEADER'='TRUE')")
    checkAnswer(sql("select id,name, salary from complexTable"),
      Seq(Row(100000001, "batch_1", 0.1),
        Row(100000002, "batch_2", 0.2),
        Row(100000003, "batch_3", 0.3),
        Row(100000004, "batch_4", 0.4),
        Row(100000005, "batch_5", 0.5)))
    sql("drop table if exists complexTable")
  }

  test("test bloom indexSchema on the adaptive encoded column") {
    sql("drop table if exists negativeTable")
    sql("create table negativeTable (intColumn int,stringColumn string,shortColumn short) " +
        "STORED AS carbondata TBLPROPERTIES('SORT_COLUMNS'='intColumn,shortColumn')")
    sql(s"load data inpath '${resourcesPath + "/dataWithNegativeValues.csv"}' " +
        s"into table negativeTable options('FILEHEADER'='intColumn,stringColumn,shortColumn')")
    checkAnswer(sql("select * from negativeTable"), sql("select * from negativeTable_Compare"))
    // create bloom in the intColumn and shortColumn
    sql("CREATE INDEX negativeTable_bloom " +
        "ON TABLE negativeTable (intColumn,shortColumn)" +
        "AS 'bloomfilter' Properties('BLOOM_SIZE'='640000')")

    checkAnswer(sql("select * from negativeTable where intColumn<0"),
      sql("select * from negativeTable_Compare where intColumn<0"))
    checkAnswer(sql("select * from negativeTable where intColumn<=0"),
      sql("select * from negativeTable_Compare where intColumn<=0"))
    checkAnswer(sql("select * from negativeTable where intColumn>0"),
      sql("select * from negativeTable_Compare where intColumn>0"))
    checkAnswer(sql("select * from negativeTable where intColumn>=0"),
      sql("select * from negativeTable_Compare where intColumn>=0"))
    checkAnswer(sql("select * from negativeTable where intColumn >= -20000"),
      sql("select * from negativeTable_Compare where intColumn >= -20000"))
    checkAnswer(sql("select * from negativeTable where intColumn <= 10000"),
      sql("select * from negativeTable_Compare where intColumn <= 10000"))
    checkAnswer(sql("select * from negativeTable where intColumn between -20000 and 10000"),
      sql("select * from negativeTable_Compare where intColumn between -20000 and 10000"))
    checkAnswer(sql("select * from negativeTable where intColumn is null"),
      sql("select * from negativeTable_Compare where intColumn is null"))
    checkAnswer(sql("select * from negativeTable where intColumn is not null"),
      sql("select * from negativeTable_Compare where intColumn is not null"))
    checkAnswer(sql("select * from negativeTable where shortColumn<0"),
      sql("select * from negativeTable_Compare where shortColumn<0"))
    checkAnswer(sql("select * from negativeTable where shortColumn<=0"),
      sql("select * from negativeTable_Compare where shortColumn<=0"))
    checkAnswer(sql("select * from negativeTable where shortColumn>0"),
      sql("select * from negativeTable_Compare where shortColumn>0"))
    checkAnswer(sql("select * from negativeTable where shortColumn>=0"),
      sql("select * from negativeTable_Compare where shortColumn>=0"))
    checkAnswer(sql("select * from negativeTable where shortColumn between -200 and 200"),
      sql("select * from negativeTable_Compare where shortColumn between -200 and 200"))
    checkAnswer(sql("select * from negativeTable where shortColumn >= -200"),
      sql("select * from negativeTable_Compare where shortColumn >= -200"))
    checkAnswer(sql("select * from negativeTable where shortColumn <= 100"),
      sql("select * from negativeTable_Compare where shortColumn <= 100"))
    checkAnswer(sql("select * from negativeTable where shortColumn is null"),
      sql("select * from negativeTable_Compare where shortColumn is null"))
    checkAnswer(sql("select * from negativeTable where shortColumn is not null"),
      sql("select * from negativeTable_Compare where shortColumn is not null"))
    sql("drop table if exists negativeTable")
  }

  test("test inverted index issue ") {
    sql("DROP TABLE IF EXISTS source")
    sql("CREATE TABLE source(intField int) stored as carbondata " +
        "TBLPROPERTIES('sort_columns'='intField','inverted_index'='intField')")
    // this is stored as short data after adaptive encoding
    sql("insert into source select 32000 union all select 0");
    checkAnswer(sql("SELECT * from source"), Seq(Row(0), Row(32000)))
    // this is stored as BYTE data after adaptive encoding
    sql("insert into source select 32000 ");
    checkAnswer(sql("SELECT * from source"), Seq(Row(0), Row(32000), Row(32000)))
    sql("DROP TABLE IF EXISTS source")
  }

  override def afterAll: Unit = {
    restoreDefaultProperty
    CarbonProperties.getInstance()
      .addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER,
        CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT)
    dropTables
  }


  private def dropTables = {
    sql("drop table if exists uniqdata")
    sql("drop table if exists uniqdata_Compare")
    sql("drop table if exists uniqdata_Compaction")
    sql("drop table if exists negativeTable")
    sql("drop table if exists negativeTable_Compare")
    sql("drop table if exists negativeTable_Compaction")
    sql("drop table if exists complexTable")
  }

  private def restoreDefaultProperty: Unit = {
    CarbonProperties.getInstance()
      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
        CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
        CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_DEFAULT)
      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
        CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE)
  }
}
