bikes.core.schemas

Define and validate dataframe schemas.

  1"""Define and validate dataframe schemas."""
  2
  3# %% IMPORTS
  4
  5import typing as T
  6
  7import pandas as pd
  8import pandera.pandas as pa
  9import pandera.typing.pandas as papd
 10from pandera.api.pandas.model_config import BaseConfig
 11
 12# %% TYPES
 13
 14# Generic type for a dataframe container
 15TSchema = T.TypeVar("TSchema", bound="pa.DataFrameModel")
 16
 17# %% SCHEMAS
 18
 19
 20class SchemaConfig(BaseConfig):
 21    """Default configurations for all schemas.
 22
 23    Parameters:
 24        coerce (bool): convert data type if possible.
 25        strict (bool): ensure the data type is correct.
 26    """
 27
 28    coerce: bool = True
 29    strict: bool = True
 30
 31
 32class Schema(pa.DataFrameModel):
 33    """Base class for a dataframe schema.
 34
 35    Use a schema to type your dataframe object.
 36    e.g., to communicate and validate its fields.
 37    """
 38
 39    Config: type[SchemaConfig] = SchemaConfig
 40
 41    @classmethod
 42    def check(cls: type[TSchema], data: pd.DataFrame) -> papd.DataFrame[TSchema]:
 43        """Check the dataframe with this schema.
 44
 45        Args:
 46            data (pd.DataFrame): dataframe to check.
 47
 48        Returns:
 49            papd.DataFrame[TSchema]: validated dataframe.
 50        """
 51        return cls.validate(data)
 52
 53    # Pandas' nullable casts reject truncation and overflow before Pandera narrows
 54    # to the public NumPy dtypes. NumPy astype alone can turn hr=256.5 into zero.
 55    @pa.parser(r"^(season|yr|mnth|hr|weekday|weathersit)$", regex=True)
 56    @classmethod
 57    def parse_uint8(cls, data: pd.Series) -> pd.Series:
 58        """Check that small integer fields can be converted without data loss."""
 59        return data.astype("UInt8")
 60
 61    @pa.parser(r"^(instant|cnt|prediction)$", regex=True)
 62    @classmethod
 63    def parse_uint32(cls, data: pd.Series) -> pd.Series:
 64        """Check that sample IDs and counts can be converted without data loss."""
 65        return data.astype("UInt32")
 66
 67    @pa.parser(r"^(holiday|workingday)$", regex=True)
 68    @classmethod
 69    def parse_boolean(cls, data: pd.Series) -> pd.Series:
 70        """Accept booleans and zero/one values without truthiness coercion."""
 71        return data.astype("boolean")
 72
 73
 74class InputsSchema(Schema):
 75    """Shared training and serving inputs, excluding target-derived rental counts."""
 76
 77    instant: papd.Index[papd.UInt32] = pa.Field(ge=0, unique=True)
 78    dteday: papd.Series[papd.DateTime] = pa.Field()
 79    season: papd.Series[papd.UInt8] = pa.Field(isin=[1, 2, 3, 4])
 80    yr: papd.Series[papd.UInt8] = pa.Field(ge=0, le=1)
 81    mnth: papd.Series[papd.UInt8] = pa.Field(ge=1, le=12)
 82    hr: papd.Series[papd.UInt8] = pa.Field(ge=0, le=23)
 83    holiday: papd.Series[papd.Bool] = pa.Field()
 84    weekday: papd.Series[papd.UInt8] = pa.Field(ge=0, le=6)
 85    workingday: papd.Series[papd.Bool] = pa.Field()
 86    weathersit: papd.Series[papd.UInt8] = pa.Field(ge=1, le=4)
 87    temp: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)
 88    atemp: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)
 89    hum: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)
 90    windspeed: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)
 91
 92
 93Inputs = papd.DataFrame[InputsSchema]
 94
 95
 96class TargetsSchema(Schema):
 97    """Schema for the project target."""
 98
 99    instant: papd.Index[papd.UInt32] = pa.Field(ge=0, unique=True)
100    cnt: papd.Series[papd.UInt32] = pa.Field(ge=0)
101
102
103Targets = papd.DataFrame[TargetsSchema]
104
105
106class OutputsSchema(Schema):
107    """Schema for the project output."""
108
109    instant: papd.Index[papd.UInt32] = pa.Field(ge=0, unique=True)
110    prediction: papd.Series[papd.UInt32] = pa.Field(ge=0)
111
112
113Outputs = papd.DataFrame[OutputsSchema]
114
115
116class SHAPValuesConfig(SchemaConfig):
117    """Default configurations this schema.
118
119    Parameters:
120        dtype (str): dataframe default data type.
121        strict (bool): ensure the data type is correct.
122    """
123
124    dtype: str = "float32"
125    strict: bool = False
126
127
128class SHAPValuesSchema(Schema):
129    """Schema for the project shap values."""
130
131    Config: type[SHAPValuesConfig] = SHAPValuesConfig
132
133
134SHAPValues = papd.DataFrame[SHAPValuesSchema]
135
136
137class FeatureImportancesSchema(Schema):
138    """Schema for the project feature importances."""
139
140    feature: papd.Series[papd.String] = pa.Field()
141    importance: papd.Series[papd.Float32] = pa.Field()
142
143
144FeatureImportances = papd.DataFrame[FeatureImportancesSchema]
class SchemaConfig(pandera.api.pandas.model_config.BaseConfig): 
21class SchemaConfig(BaseConfig):
22    """Default configurations for all schemas.
23
24    Parameters:
25        coerce (bool): convert data type if possible.
26        strict (bool): ensure the data type is correct.
27    """
28
29    coerce: bool = True
30    strict: bool = True

Default configurations for all schemas.

Arguments:
  • coerce (bool): convert data type if possible.
  • strict (bool): ensure the data type is correct.
coerce: bool = True
strict: bool = True
name: str | None = None
class Schema(typing.Generic[~TDataFrame, ~TSchema], pandera.api.base.model.BaseModel): 
33class Schema(pa.DataFrameModel):
34    """Base class for a dataframe schema.
35
36    Use a schema to type your dataframe object.
37    e.g., to communicate and validate its fields.
38    """
39
40    Config: type[SchemaConfig] = SchemaConfig
41
42    @classmethod
43    def check(cls: type[TSchema], data: pd.DataFrame) -> papd.DataFrame[TSchema]:
44        """Check the dataframe with this schema.
45
46        Args:
47            data (pd.DataFrame): dataframe to check.
48
49        Returns:
50            papd.DataFrame[TSchema]: validated dataframe.
51        """
52        return cls.validate(data)
53
54    # Pandas' nullable casts reject truncation and overflow before Pandera narrows
55    # to the public NumPy dtypes. NumPy astype alone can turn hr=256.5 into zero.
56    @pa.parser(r"^(season|yr|mnth|hr|weekday|weathersit)$", regex=True)
57    @classmethod
58    def parse_uint8(cls, data: pd.Series) -> pd.Series:
59        """Check that small integer fields can be converted without data loss."""
60        return data.astype("UInt8")
61
62    @pa.parser(r"^(instant|cnt|prediction)$", regex=True)
63    @classmethod
64    def parse_uint32(cls, data: pd.Series) -> pd.Series:
65        """Check that sample IDs and counts can be converted without data loss."""
66        return data.astype("UInt32")
67
68    @pa.parser(r"^(holiday|workingday)$", regex=True)
69    @classmethod
70    def parse_boolean(cls, data: pd.Series) -> pd.Series:
71        """Accept booleans and zero/one values without truthiness coercion."""
72        return data.astype("boolean")

Base class for a dataframe schema.

Use a schema to type your dataframe object. e.g., to communicate and validate its fields.

@docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
Schema(*args, **kwargs) 
283    @docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
284    def __new__(cls, *args, **kwargs) -> DataFrameBase[Self]:  # type: ignore [misc]
285        """%(validate_doc)s"""
286        return cast(DataFrameBase[Self], cls.validate(*args, **kwargs))

Validate a DataFrame based on the schema specification.

Parameters
  • pd.DataFrame check_obj: the dataframe to be validated.
  • head: validate the first n rows. Rows overlapping with tail or sample are de-duplicated.
  • tail: validate the last n rows. Rows overlapping with head or sample are de-duplicated.
  • sample: validate a random sample of n rows. Rows overlapping with head or tail are de-duplicated.
  • random_state: random seed for the sample argument.
  • lazy: if True, lazily evaluates dataframe against all validation checks and raises a SchemaErrors. Otherwise, raise SchemaError as soon as one occurs.
  • inplace: if True, applies coercion to the object of validation, otherwise creates a copy of the data. :returns: validated DataFrame
Raises
  • SchemaError: when DataFrame violates built-in or custom checks.
@classmethod
def check( cls: type[~TSchema], data: pandas.DataFrame) -> pandera.typing.pandas.DataFrame[~TSchema]: 
42    @classmethod
43    def check(cls: type[TSchema], data: pd.DataFrame) -> papd.DataFrame[TSchema]:
44        """Check the dataframe with this schema.
45
46        Args:
47            data (pd.DataFrame): dataframe to check.
48
49        Returns:
50            papd.DataFrame[TSchema]: validated dataframe.
51        """
52        return cls.validate(data)

Check the dataframe with this schema.

Arguments:
  • data (pd.DataFrame): dataframe to check.
Returns:

papd.DataFrame[TSchema]: validated dataframe.

@pa.parser('^(season|yr|mnth|hr|weekday|weathersit)$', regex=True)
@classmethod
def parse_uint8(cls, data: pandas.Series) -> pandas.Series: 
56    @pa.parser(r"^(season|yr|mnth|hr|weekday|weathersit)$", regex=True)
57    @classmethod
58    def parse_uint8(cls, data: pd.Series) -> pd.Series:
59        """Check that small integer fields can be converted without data loss."""
60        return data.astype("UInt8")

Check that small integer fields can be converted without data loss.

@pa.parser('^(instant|cnt|prediction)$', regex=True)
@classmethod
def parse_uint32(cls, data: pandas.Series) -> pandas.Series: 
62    @pa.parser(r"^(instant|cnt|prediction)$", regex=True)
63    @classmethod
64    def parse_uint32(cls, data: pd.Series) -> pd.Series:
65        """Check that sample IDs and counts can be converted without data loss."""
66        return data.astype("UInt32")

Check that sample IDs and counts can be converted without data loss.

@pa.parser('^(holiday|workingday)$', regex=True)
@classmethod
def parse_boolean(cls, data: pandas.Series) -> pandas.Series: 
68    @pa.parser(r"^(holiday|workingday)$", regex=True)
69    @classmethod
70    def parse_boolean(cls, data: pd.Series) -> pd.Series:
71        """Accept booleans and zero/one values without truthiness coercion."""
72        return data.astype("boolean")

Accept booleans and zero/one values without truthiness coercion.

class InputsSchema(typing.Generic[~TDataFrame, ~TSchema], pandera.api.base.model.BaseModel): 
75class InputsSchema(Schema):
76    """Shared training and serving inputs, excluding target-derived rental counts."""
77
78    instant: papd.Index[papd.UInt32] = pa.Field(ge=0, unique=True)
79    dteday: papd.Series[papd.DateTime] = pa.Field()
80    season: papd.Series[papd.UInt8] = pa.Field(isin=[1, 2, 3, 4])
81    yr: papd.Series[papd.UInt8] = pa.Field(ge=0, le=1)
82    mnth: papd.Series[papd.UInt8] = pa.Field(ge=1, le=12)
83    hr: papd.Series[papd.UInt8] = pa.Field(ge=0, le=23)
84    holiday: papd.Series[papd.Bool] = pa.Field()
85    weekday: papd.Series[papd.UInt8] = pa.Field(ge=0, le=6)
86    workingday: papd.Series[papd.Bool] = pa.Field()
87    weathersit: papd.Series[papd.UInt8] = pa.Field(ge=1, le=4)
88    temp: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)
89    atemp: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)
90    hum: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)
91    windspeed: papd.Series[papd.Float16] = pa.Field(ge=0, le=1)

Shared training and serving inputs, excluding target-derived rental counts.

@docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
InputsSchema(*args, **kwargs) 
283    @docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
284    def __new__(cls, *args, **kwargs) -> DataFrameBase[Self]:  # type: ignore [misc]
285        """%(validate_doc)s"""
286        return cast(DataFrameBase[Self], cls.validate(*args, **kwargs))

Validate a DataFrame based on the schema specification.

Parameters
  • pd.DataFrame check_obj: the dataframe to be validated.
  • head: validate the first n rows. Rows overlapping with tail or sample are de-duplicated.
  • tail: validate the last n rows. Rows overlapping with head or sample are de-duplicated.
  • sample: validate a random sample of n rows. Rows overlapping with head or tail are de-duplicated.
  • random_state: random seed for the sample argument.
  • lazy: if True, lazily evaluates dataframe against all validation checks and raises a SchemaErrors. Otherwise, raise SchemaError as soon as one occurs.
  • inplace: if True, applies coercion to the object of validation, otherwise creates a copy of the data. :returns: validated DataFrame
Raises
  • SchemaError: when DataFrame violates built-in or custom checks.
instant: pandera.typing.pandas.Index[pandera.dtypes.UInt32]

Captures extra information about a field.

new in 0.5.0

dteday: pandera.typing.pandas.Series[pandera.dtypes.Timestamp]

Captures extra information about a field.

new in 0.5.0

season: pandera.typing.pandas.Series[pandera.dtypes.UInt8]

Captures extra information about a field.

new in 0.5.0

yr: pandera.typing.pandas.Series[pandera.dtypes.UInt8]

Captures extra information about a field.

new in 0.5.0

mnth: pandera.typing.pandas.Series[pandera.dtypes.UInt8]

Captures extra information about a field.

new in 0.5.0

hr: pandera.typing.pandas.Series[pandera.dtypes.UInt8]

Captures extra information about a field.

new in 0.5.0

holiday: pandera.typing.pandas.Series[pandera.dtypes.Bool]

Captures extra information about a field.

new in 0.5.0

weekday: pandera.typing.pandas.Series[pandera.dtypes.UInt8]

Captures extra information about a field.

new in 0.5.0

workingday: pandera.typing.pandas.Series[pandera.dtypes.Bool]

Captures extra information about a field.

new in 0.5.0

weathersit: pandera.typing.pandas.Series[pandera.dtypes.UInt8]

Captures extra information about a field.

new in 0.5.0

temp: pandera.typing.pandas.Series[pandera.dtypes.Float16]

Captures extra information about a field.

new in 0.5.0

atemp: pandera.typing.pandas.Series[pandera.dtypes.Float16]

Captures extra information about a field.

new in 0.5.0

hum: pandera.typing.pandas.Series[pandera.dtypes.Float16]

Captures extra information about a field.

new in 0.5.0

windspeed: pandera.typing.pandas.Series[pandera.dtypes.Float16]

Captures extra information about a field.

new in 0.5.0

Inherited Members
Schema
check
parse_uint8
parse_uint32
parse_boolean
Inputs = pandera.typing.pandas.DataFrame[InputsSchema]
class TargetsSchema(typing.Generic[~TDataFrame, ~TSchema], pandera.api.base.model.BaseModel): 
 97class TargetsSchema(Schema):
 98    """Schema for the project target."""
 99
100    instant: papd.Index[papd.UInt32] = pa.Field(ge=0, unique=True)
101    cnt: papd.Series[papd.UInt32] = pa.Field(ge=0)

Schema for the project target.

@docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
TargetsSchema(*args, **kwargs) 
283    @docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
284    def __new__(cls, *args, **kwargs) -> DataFrameBase[Self]:  # type: ignore [misc]
285        """%(validate_doc)s"""
286        return cast(DataFrameBase[Self], cls.validate(*args, **kwargs))

Validate a DataFrame based on the schema specification.

Parameters
  • pd.DataFrame check_obj: the dataframe to be validated.
  • head: validate the first n rows. Rows overlapping with tail or sample are de-duplicated.
  • tail: validate the last n rows. Rows overlapping with head or sample are de-duplicated.
  • sample: validate a random sample of n rows. Rows overlapping with head or tail are de-duplicated.
  • random_state: random seed for the sample argument.
  • lazy: if True, lazily evaluates dataframe against all validation checks and raises a SchemaErrors. Otherwise, raise SchemaError as soon as one occurs.
  • inplace: if True, applies coercion to the object of validation, otherwise creates a copy of the data. :returns: validated DataFrame
Raises
  • SchemaError: when DataFrame violates built-in or custom checks.
instant: pandera.typing.pandas.Index[pandera.dtypes.UInt32]

Captures extra information about a field.

new in 0.5.0

cnt: pandera.typing.pandas.Series[pandera.dtypes.UInt32]

Captures extra information about a field.

new in 0.5.0

Inherited Members
Schema
check
parse_uint8
parse_uint32
parse_boolean
Targets = pandera.typing.pandas.DataFrame[TargetsSchema]
class OutputsSchema(typing.Generic[~TDataFrame, ~TSchema], pandera.api.base.model.BaseModel): 
107class OutputsSchema(Schema):
108    """Schema for the project output."""
109
110    instant: papd.Index[papd.UInt32] = pa.Field(ge=0, unique=True)
111    prediction: papd.Series[papd.UInt32] = pa.Field(ge=0)

Schema for the project output.

@docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
OutputsSchema(*args, **kwargs) 
283    @docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
284    def __new__(cls, *args, **kwargs) -> DataFrameBase[Self]:  # type: ignore [misc]
285        """%(validate_doc)s"""
286        return cast(DataFrameBase[Self], cls.validate(*args, **kwargs))

Validate a DataFrame based on the schema specification.

Parameters
  • pd.DataFrame check_obj: the dataframe to be validated.
  • head: validate the first n rows. Rows overlapping with tail or sample are de-duplicated.
  • tail: validate the last n rows. Rows overlapping with head or sample are de-duplicated.
  • sample: validate a random sample of n rows. Rows overlapping with head or tail are de-duplicated.
  • random_state: random seed for the sample argument.
  • lazy: if True, lazily evaluates dataframe against all validation checks and raises a SchemaErrors. Otherwise, raise SchemaError as soon as one occurs.
  • inplace: if True, applies coercion to the object of validation, otherwise creates a copy of the data. :returns: validated DataFrame
Raises
  • SchemaError: when DataFrame violates built-in or custom checks.
instant: pandera.typing.pandas.Index[pandera.dtypes.UInt32]

Captures extra information about a field.

new in 0.5.0

prediction: pandera.typing.pandas.Series[pandera.dtypes.UInt32]

Captures extra information about a field.

new in 0.5.0

Inherited Members
Schema
check
parse_uint8
parse_uint32
parse_boolean
Outputs = pandera.typing.pandas.DataFrame[OutputsSchema]
class SHAPValuesConfig(SchemaConfig): 
117class SHAPValuesConfig(SchemaConfig):
118    """Default configurations this schema.
119
120    Parameters:
121        dtype (str): dataframe default data type.
122        strict (bool): ensure the data type is correct.
123    """
124
125    dtype: str = "float32"
126    strict: bool = False

Default configurations this schema.

Arguments:
  • dtype (str): dataframe default data type.
  • strict (bool): ensure the data type is correct.
dtype: str = 'float32'
strict: bool = False
name: str | None = None
Inherited Members
SchemaConfig
coerce
class SHAPValuesSchema(typing.Generic[~TDataFrame, ~TSchema], pandera.api.base.model.BaseModel): 
129class SHAPValuesSchema(Schema):
130    """Schema for the project shap values."""
131
132    Config: type[SHAPValuesConfig] = SHAPValuesConfig

Schema for the project shap values.

@docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
SHAPValuesSchema(*args, **kwargs) 
283    @docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
284    def __new__(cls, *args, **kwargs) -> DataFrameBase[Self]:  # type: ignore [misc]
285        """%(validate_doc)s"""
286        return cast(DataFrameBase[Self], cls.validate(*args, **kwargs))

Validate a DataFrame based on the schema specification.

Parameters
  • pd.DataFrame check_obj: the dataframe to be validated.
  • head: validate the first n rows. Rows overlapping with tail or sample are de-duplicated.
  • tail: validate the last n rows. Rows overlapping with head or sample are de-duplicated.
  • sample: validate a random sample of n rows. Rows overlapping with head or tail are de-duplicated.
  • random_state: random seed for the sample argument.
  • lazy: if True, lazily evaluates dataframe against all validation checks and raises a SchemaErrors. Otherwise, raise SchemaError as soon as one occurs.
  • inplace: if True, applies coercion to the object of validation, otherwise creates a copy of the data. :returns: validated DataFrame
Raises
  • SchemaError: when DataFrame violates built-in or custom checks.
Inherited Members
Schema
check
parse_uint8
parse_uint32
parse_boolean
SHAPValues = pandera.typing.pandas.DataFrame[SHAPValuesSchema]
class FeatureImportancesSchema(typing.Generic[~TDataFrame, ~TSchema], pandera.api.base.model.BaseModel): 
138class FeatureImportancesSchema(Schema):
139    """Schema for the project feature importances."""
140
141    feature: papd.Series[papd.String] = pa.Field()
142    importance: papd.Series[papd.Float32] = pa.Field()

Schema for the project feature importances.

@docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
FeatureImportancesSchema(*args, **kwargs) 
283    @docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
284    def __new__(cls, *args, **kwargs) -> DataFrameBase[Self]:  # type: ignore [misc]
285        """%(validate_doc)s"""
286        return cast(DataFrameBase[Self], cls.validate(*args, **kwargs))

Validate a DataFrame based on the schema specification.

Parameters
  • pd.DataFrame check_obj: the dataframe to be validated.
  • head: validate the first n rows. Rows overlapping with tail or sample are de-duplicated.
  • tail: validate the last n rows. Rows overlapping with head or sample are de-duplicated.
  • sample: validate a random sample of n rows. Rows overlapping with head or tail are de-duplicated.
  • random_state: random seed for the sample argument.
  • lazy: if True, lazily evaluates dataframe against all validation checks and raises a SchemaErrors. Otherwise, raise SchemaError as soon as one occurs.
  • inplace: if True, applies coercion to the object of validation, otherwise creates a copy of the data. :returns: validated DataFrame
Raises
  • SchemaError: when DataFrame violates built-in or custom checks.
feature: pandera.typing.pandas.Series[pandera.dtypes.String]

Captures extra information about a field.

new in 0.5.0

importance: pandera.typing.pandas.Series[pandera.dtypes.Float32]

Captures extra information about a field.

new in 0.5.0

Inherited Members
Schema
check
parse_uint8
parse_uint32
parse_boolean
FeatureImportances = pandera.typing.pandas.DataFrame[FeatureImportancesSchema]