diff --git a/sail-partitioned/install b/sail-partitioned/install index 3249b4e982..78db67204f 100755 --- a/sail-partitioned/install +++ b/sail-partitioned/install @@ -49,5 +49,5 @@ source myenv/bin/activate # RECORD file, so `--upgrade` fails to uninstall it. `--ignore-installed` # avoids the uninstall and just lays a fresh copy in the venv. pip install --upgrade --ignore-installed setuptools wheel -pip install --no-cache-dir "pysail==0.5.2" +pip install --no-cache-dir "pysail==0.6.5" pip install "pyspark-client==4.1.1" pandas psutil diff --git a/sail-partitioned/query b/sail-partitioned/query index ae1546f7ab..06afad5d97 100755 --- a/sail-partitioned/query +++ b/sail-partitioned/query @@ -28,6 +28,7 @@ os.environ["SAIL_OPTIMIZER__ENABLE_JOIN_REORDER"] = "true" from pysail.spark import SparkConnectServer from pyspark.sql import SparkSession +import pyspark.sql.functions as F with open(sys.argv[1]) as f: query = f.read() @@ -39,6 +40,8 @@ _, port = server.listening_address spark = SparkSession.builder.remote(f"sc://localhost:{port}").getOrCreate() df = spark.read.parquet("partitioned") +df = df.withColumn("EventTime", F.col("EventTime").cast("timestamp")) +df = df.withColumn("EventDate", F.col("EventDate").cast("int").cast("date")) df.createOrReplaceTempView("hits") start = timeit.default_timer() diff --git a/sail/install b/sail/install index 4aa2a2411a..24a9df6090 100755 --- a/sail/install +++ b/sail/install @@ -55,5 +55,5 @@ source myenv/bin/activate # `--ignore-installed` skips the uninstall and just lays the new copy on # top, which is what we want inside this venv anyway. pip install --upgrade --ignore-installed setuptools wheel -pip install --no-cache-dir "pysail==0.5.2" +pip install --no-cache-dir "pysail==0.6.5" pip install "pyspark-client==4.1.1" pandas psutil diff --git a/sail/query b/sail/query index 51e3704097..297c07e9e0 100755 --- a/sail/query +++ b/sail/query @@ -28,6 +28,7 @@ os.environ["SAIL_OPTIMIZER__ENABLE_JOIN_REORDER"] = "true" from pysail.spark import SparkConnectServer from pyspark.sql import SparkSession +import pyspark.sql.functions as F with open(sys.argv[1]) as f: query = f.read() @@ -40,6 +41,8 @@ _, port = server.listening_address spark = SparkSession.builder.remote(f"sc://localhost:{port}").getOrCreate() df = spark.read.parquet("hits.parquet") +df = df.withColumn("EventTime", F.col("EventTime").cast("timestamp")) +df = df.withColumn("EventDate", F.col("EventDate").cast("int").cast("date")) df.createOrReplaceTempView("hits") start = timeit.default_timer()