1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
import pandas as pd
from toolz import curried
from ..utils.core import sanitize_dataframe
from ..utils.data import (
MaxRowsError,
curry,
pipe,
sample,
to_csv,
to_json,
to_values,
check_data_type,
)
@curried.curry
def limit_rows(data, max_rows=5000):
"""Raise MaxRowsError if the data model has more than max_rows."""
if not isinstance(data, (list, pd.DataFrame)):
raise TypeError("Expected dict or DataFrame, got: {}".format(type(data)))
if len(data) > max_rows:
raise MaxRowsError(
"The number of rows in your dataset is greater than the max of {}".format(
max_rows
)
)
return data
@curried.curry
def default_data_transformer(data):
return curried.pipe(data, limit_rows, to_values)
__all__ = (
"MaxRowsError",
"curry",
"default_data_transformer",
"limit_rows",
"pipe",
"sanitize_dataframe",
"sample",
"to_csv",
"to_json",
"to_values",
"check_data_type",
)
|
