Skip to content

pandas_openscm.db.feather#

Feather backend

Classes:

Name Description
FeatherDataBackend

Feather data backend
FeatherIndexBackend

Feather index backend

FeatherDataBackend #

Feather data backend

For details on feather, see https://arrow.apache.org/docs/python/feather.html

Methods:

Name Description
load_data

Load a data file
save_data

Save data to disk

Attributes:

Name Type Description
ext str

Extension to use with files saved by this backend.
preserves_index Literal[True]

Whether this backend preserves the index of data upon (de-)serialisation
Source code in src/pandas_openscm/db/feather.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
@define
class FeatherDataBackend:
    """
    Feather data backend

    For details on feather, see https://arrow.apache.org/docs/python/feather.html
    """

    ext: str = ".feather"
    """
    Extension to use with files saved by this backend.
    """

    @property
    def preserves_index(self) -> Literal[True]:
        """
        Whether this backend preserves the index of data upon (de-)serialisation
        """
        return True

    @staticmethod
    def load_data(data_file: Path) -> pd.DataFrame:
        """
        Load a data file

        Parameters
        ----------
        data_file
            File from which to load the data

        Returns
        -------
        :
            Loaded data
        """
        return pd.read_feather(data_file)

    @staticmethod
    def save_data(data: pd.DataFrame, data_file: Path) -> None:
        """
        Save data to disk

        Parameters
        ----------
        data
            Data to save

        data_file
            File in which to save the data
        """
        # The docs say that feather doesn't support writing indexes
        # # (see https://pandas.pydata.org/docs/user_guide/io.html#feather).
        # However, it seems to have no issue writing our multi-indexes.
        # Hence the implementation below
        data.to_feather(data_file)

ext class-attribute instance-attribute #

ext: str = '.feather'

Extension to use with files saved by this backend.

preserves_index property #

preserves_index: Literal[True]

Whether this backend preserves the index of data upon (de-)serialisation

load_data staticmethod #

load_data(data_file: Path) -> DataFrame

Load a data file

Parameters:

Name Type Description Default
data_file Path

File from which to load the data

 required

Returns:

Type Description
DataFrame

Loaded data
Source code in src/pandas_openscm/db/feather.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
@staticmethod
def load_data(data_file: Path) -> pd.DataFrame:
    """
    Load a data file

    Parameters
    ----------
    data_file
        File from which to load the data

    Returns
    -------
    :
        Loaded data
    """
    return pd.read_feather(data_file)

save_data staticmethod #

save_data(data: DataFrame, data_file: Path) -> None

Save data to disk

Parameters:

Name Type Description Default
data DataFrame

Data to save

 required
data_file Path

File in which to save the data

 required
Source code in src/pandas_openscm/db/feather.py 
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
@staticmethod
def save_data(data: pd.DataFrame, data_file: Path) -> None:
    """
    Save data to disk

    Parameters
    ----------
    data
        Data to save

    data_file
        File in which to save the data
    """
    # The docs say that feather doesn't support writing indexes
    # # (see https://pandas.pydata.org/docs/user_guide/io.html#feather).
    # However, it seems to have no issue writing our multi-indexes.
    # Hence the implementation below
    data.to_feather(data_file)

FeatherIndexBackend #

Feather index backend

For details on feather, see https://arrow.apache.org/docs/python/feather.html

Methods:

Name Description
load_file_map

Load the file map
load_index

Load the index
save_file_map

Save the file map to disk
save_index

Save the index to disk

Attributes:

Name Type Description
ext str

Extension to use with files saved by this backend.
preserves_index Literal[True]

Whether this backend preserves the pd.MultiIndex upon (de-)serialisation
Source code in src/pandas_openscm/db/feather.py 
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
@define
class FeatherIndexBackend:
    """
    Feather index backend

    For details on feather, see https://arrow.apache.org/docs/python/feather.html
    """

    ext: str = ".feather"
    """
    Extension to use with files saved by this backend.
    """

    @property
    def preserves_index(self) -> Literal[True]:
        """
        Whether this backend preserves the `pd.MultiIndex` upon (de-)serialisation
        """
        return True

    @staticmethod
    def load_file_map(file_map_file: Path) -> pd.DataFrame:
        """
        Load the file map

        Parameters
        ----------
        file_map_file
            File from which to load the file map

        Returns
        -------
        :
            Loaded file map
        """
        return pd.read_feather(file_map_file)

    @staticmethod
    def load_index(index_file: Path) -> pd.DataFrame:
        """
        Load the index

        Parameters
        ----------
        index_file
            File from which to load the index

        Returns
        -------
        :
            Loaded index
        """
        return pd.read_feather(index_file)

    @staticmethod
    def save_file_map(
        file_map: pd.Series[Path],  # type: ignore # pandas confused about what it supports
        file_map_file: Path,
    ) -> None:
        """
        Save the file map to disk

        Parameters
        ----------
        file_map
            File map to save

        file_map_file
            File in which to save the file map
        """
        # Feather doesn't support writing non-native types
        # (see https://pandas.pydata.org/docs/user_guide/io.html#feather).
        # The docs say that feather doesn't support writing indexes
        # # (see https://pandas.pydata.org/docs/user_guide/io.html#feather).
        # However, it seems to have no issue writing this index.
        # Hence the implementation below
        file_map_write = file_map.astype(str)
        file_map_write.to_frame().to_feather(file_map_file)

    @staticmethod
    def save_index(
        index: pd.DataFrame,
        index_file: Path,
    ) -> None:
        """
        Save the index to disk

        Parameters
        ----------
        index
            Index to save

        index_file
            File in which to save the index
        """
        index.to_feather(index_file)

ext class-attribute instance-attribute #

ext: str = '.feather'

Extension to use with files saved by this backend.

preserves_index property #

preserves_index: Literal[True]

Whether this backend preserves the pd.MultiIndex upon (de-)serialisation

load_file_map staticmethod #

load_file_map(file_map_file: Path) -> DataFrame

Load the file map

Parameters:

Name Type Description Default
file_map_file Path

File from which to load the file map

 required

Returns:

Type Description
DataFrame

Loaded file map
Source code in src/pandas_openscm/db/feather.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
@staticmethod
def load_file_map(file_map_file: Path) -> pd.DataFrame:
    """
    Load the file map

    Parameters
    ----------
    file_map_file
        File from which to load the file map

    Returns
    -------
    :
        Loaded file map
    """
    return pd.read_feather(file_map_file)

load_index staticmethod #

load_index(index_file: Path) -> DataFrame

Load the index

Parameters:

Name Type Description Default
index_file Path

File from which to load the index

 required

Returns:

Type Description
DataFrame

Loaded index
Source code in src/pandas_openscm/db/feather.py 
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
@staticmethod
def load_index(index_file: Path) -> pd.DataFrame:
    """
    Load the index

    Parameters
    ----------
    index_file
        File from which to load the index

    Returns
    -------
    :
        Loaded index
    """
    return pd.read_feather(index_file)

save_file_map staticmethod #

save_file_map(
    file_map: Series[Path], file_map_file: Path
) -> None

Save the file map to disk

Parameters:

Name Type Description Default
file_map Series[Path]

File map to save

 required
file_map_file Path

File in which to save the file map

 required
Source code in src/pandas_openscm/db/feather.py 
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
@staticmethod
def save_file_map(
    file_map: pd.Series[Path],  # type: ignore # pandas confused about what it supports
    file_map_file: Path,
) -> None:
    """
    Save the file map to disk

    Parameters
    ----------
    file_map
        File map to save

    file_map_file
        File in which to save the file map
    """
    # Feather doesn't support writing non-native types
    # (see https://pandas.pydata.org/docs/user_guide/io.html#feather).
    # The docs say that feather doesn't support writing indexes
    # # (see https://pandas.pydata.org/docs/user_guide/io.html#feather).
    # However, it seems to have no issue writing this index.
    # Hence the implementation below
    file_map_write = file_map.astype(str)
    file_map_write.to_frame().to_feather(file_map_file)

save_index staticmethod #

save_index(index: DataFrame, index_file: Path) -> None

Save the index to disk

Parameters:

Name Type Description Default
index DataFrame

Index to save

 required
index_file Path

File in which to save the index

 required
Source code in src/pandas_openscm/db/feather.py 
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
@staticmethod
def save_index(
    index: pd.DataFrame,
    index_file: Path,
) -> None:
    """
    Save the index to disk

    Parameters
    ----------
    index
        Index to save

    index_file
        File in which to save the index
    """
    index.to_feather(index_file)