parquet-python | python implementation of the parquet columnar file format | Data Manipulation library

import ast

df.astype({'col_set': str}).to_parquet('data.parquet')
df1 = pd.read_parquet('data.parquet') \
        .assign(col_set=lambda x: x['col_set'].map(ast.literal_eval))
print(df1)

# Output
     col_set
0  {C, B, A}
1  {F, E, D}

In [9]: tinydf = pd.DataFrame({"col1": [11, 21], "col2": [12, 22]})
   ...: for i in range(1000):
   ...:     tinydf.to_parquet(f"myfile_{i}.parquet")

In [10]: df = dask.dataframe.read_parquet([f"myfile_{i}.parquet

def my_function(dfx):
    # return dfx['abc'] = dfx['def'] + 1
    # the above returns the result of assignment
    # we need to separate the assignment and return statements
    dfx['abc'] = dfx['def'] + 1
    return dfx

df = dd.read_par

def worker(i):
    from time import sleep
    print(f"working on {i}")
    sleep(2)

if __name__ == "__main__":
    from concurrent.futures import ThreadPoolExecutor
    for i in range(10):
        with ThreadPoolExecutor() as ex:
        

df = spark_read.format('delta').load(location) \
  .filter("date = '20221209' and object = 34")

df = spark_read.format('delta').load(location)
folder_partition = '/date=20221209/object=34'.split("/")
cols = [f"{s[0

VTS,2010-02-16 08:02:00,2010-02-16 08:14:00,5,4.2999999999999998,-73.955112999999997,40.786718,1,,-73.924710000000005,40.841335000000001,CSH,11.699999999999999,0,0.5,0,0,12.199999999999999
CMT,2010-02-24 16:25:18,2010-02-24 16:52:14,1,12.4

# test.py
import unittest
from unittest.mock import patch, PropertyMock, Mock

from pyspark.sql import SparkSession, DataFrame, functions as f
from pyspark_test import assert_pyspark_df_equal


class ClassToTest:
    def __init__(self) -&g