nsedt.utils.data_format

return data in specific format

View Source

  1"""
  2return data in specific format
  3"""
  4
  5import pandas as pd
  6
  7
  8def price(result):
  9    """
 10    Args:
 11        result (Pandas DataFrame): result
 12
 13    Returns:
 14        Pandas DataFrame: df containing data in specific format
 15    """
 16
 17    columns_required = [
 18        "CH_TIMESTAMP",
 19        "CH_OPENING_PRICE",
 20        "CH_TRADE_HIGH_PRICE",
 21        "CH_TRADE_LOW_PRICE",
 22        "CH_CLOSING_PRICE",
 23        "CH_PREVIOUS_CLS_PRICE",
 24        "CH_LAST_TRADED_PRICE",
 25        "CH_TOT_TRADED_QTY",
 26        "CH_TOT_TRADED_VAL",
 27        "CH_52WEEK_HIGH_PRICE",
 28        "CH_52WEEK_LOW_PRICE",
 29        "VWAP",
 30        "COP_DELIV_QTY",
 31        "COP_DELIV_PERC",
 32        "CH_SERIES",
 33    ]
 34
 35    try:
 36        result = result[columns_required]
 37    except:  # pylint: disable=W0702
 38        return result
 39    result = result.set_axis(
 40        [
 41            "Date",
 42            "Open Price",
 43            "High Price",
 44            "Low Price",
 45            "Close Price",
 46            "Prev Close Price",
 47            "Last Traded Price",
 48            "Total Traded Quantity",
 49            "Total Traded Value",
 50            "52 Week High Price",
 51            "52 Week Low Price",
 52            "VWAP",
 53            "Deliverable Volume",
 54            "Deliverable Percent",
 55            "Series",
 56        ],
 57        axis=1,
 58    )
 59
 60    result["Date"] = pd.to_datetime(result["Date"])
 61    result = result.sort_values("Date", ascending=True)
 62    result.reset_index(drop=True, inplace=True)
 63    return result
 64
 65
 66def indices(
 67    data_json,
 68    columns_drop_list: list = None,
 69    columns_rename_map: map = None,
 70):
 71    """
 72    Args:
 73        data_json (json):  data in json format
 74    Returns:
 75        Pandas DataFrame: df with indexCloseOnlineRecords and indexTurnoverRecords
 76    """
 77    if columns_drop_list:
 78        columns_list = columns_drop_list
 79    else:
 80        columns_list = ["_id", "EOD_INDEX_NAME", "TIMESTAMP"]
 81
 82    if columns_rename_map:
 83        columns_rename = columns_rename_map
 84    else:
 85        columns_rename = {
 86            "EOD_OPEN_INDEX_VAL": "Open Price",
 87            "EOD_HIGH_INDEX_VAL": "High Price",
 88            "EOD_CLOSE_INDEX_VAL": "Close Price",
 89            "EOD_LOW_INDEX_VAL": "Low Price",
 90            "EOD_TIMESTAMP": "Date",
 91        }
 92    data_close_df = (
 93        pd.DataFrame(data_json["data"]["indexCloseOnlineRecords"])
 94        .drop(columns=columns_list)
 95        .rename(columns=columns_rename)
 96    )
 97
 98    ## Mismatch values
 99    # data_turnover_df = (
100    #     pd.DataFrame(d["data"]["indexTurnoverRecords"])
101    #     .drop(columns=["_id", "HIT_INDEX_NAME_UPPER", "HIT_TIMESTAMP"])
102    #     .rename(
103    #         columns={
104    #             "HIT_TRADED_QTY": "Total Traded Quantity",
105    #             "HIT_TURN_OVER": "Total Traded Value",
106    #             "TIMESTAMP": "Date",
107    #         }
108    #     )
109    # )
110    return data_close_df
111    # return pd.merge(data_close_df, data_turnover_df, on="Date", how="inner")
112
113
114def option_chain(
115    data_json: str,
116    response_type: str,
117):
118    """_summary_
119
120    Args:
121        data_json (str): _description_
122        response_type (str): _description_
123
124    Returns:
125        _type_: _description_
126    """
127    if response_type == "json":
128        data_json_ret = []
129        for record in data_json:
130            if "PE" in record:
131                record["PE"].pop("strikePrice", None)
132                record["PE"].pop("expiryDate", None)
133                record["PE"].pop("underlying", None)
134                record["PE"].pop("identifier", None)
135            if "CE" in record:
136                record["CE"].pop("strikePrice", None)
137                record["CE"].pop("expiryDate", None)
138                record["CE"].pop("underlying", None)
139                record["CE"].pop("identifier", None)
140            data_json_ret.append(record)
141        return data_json_ret
142
143    return (
144        pd.json_normalize(data_json)
145        .sort_values(by=["expiryDate", "strikePrice"], ascending=True)
146        .drop(
147            columns=[
148                "PE.strikePrice",
149                "PE.expiryDate",
150                "PE.identifier",
151                "CE.strikePrice",
152                "CE.expiryDate",
153                "CE.identifier",
154            ]
155        )
156    )
157
158
159def get_vix(
160    data_json: object,
161    response_type: str = "panda_df",
162    columns_drop_list: list = None,
163):
164    """
165        Format Vix data
166    Args:
167        data_json (object): data in json format.
168        response_type (str, optional): response_type. Defaults to "panda_df".
169        columns_drop_list (list, optional): custom columns drop list. Defaults to None.
170    Returns:
171        _type_: _description_
172    """
173    data_json = data_json["data"]
174    if columns_drop_list:
175        columns_list = columns_drop_list
176    else:
177        columns_list = [
178            "_id",
179            "TIMESTAMP",
180            "createdAt",
181            "updatedAt",
182            "__v",
183            "ALTERNATE_INDEX_NAME",
184            "EOD_INDEX_NAME",
185            "EOD_PREV_CLOSE",
186            "VIX_PTS_CHG",
187            "VIX_PERC_CHG",
188        ]
189    if response_type == "json":
190        data_json_ret = []
191        for record in data_json:
192            for column in columns_list:
193                record.pop(column, None)
194
195            data_json_ret.append(record)
196        return data_json_ret
197
198    return (
199        pd.json_normalize(data_json)
200        .drop(columns=columns_list)
201        .rename(
202            columns={
203                "EOD_OPEN_INDEX_VAL": "Open Price",
204                "EOD_HIGH_INDEX_VAL": "High Price",
205                "EOD_CLOSE_INDEX_VAL": "Close Price",
206                "EOD_LOW_INDEX_VAL": "Low Price",
207                "EOD_TIMESTAMP": "Date",
208            }
209        )
210    )
211
212
213def derivatives_futures(
214    data_json: str,
215    response_type: str = "panda_df",
216    columns_drop_list=None,
217):
218    """
219        Format futures data
220
221    Args:
222        data_json (object): data in json format.
223        response_type (str, optional): response_type. Defaults to "panda_df".
224        columns_drop_list (list, optional): custom columns drop list. Defaults to None.
225
226    Returns:
227            json: format data in json
228        or
229            dataframe: format data in panda df
230    """
231    if columns_drop_list:
232        columns_list = columns_drop_list
233    else:
234        columns_list = [
235            "_id",
236            "FH_MARKET_LOT",
237            "FH_MARKET_TYPE",
238            "FH_OPTION_TYPE",
239            "FH_SYMBOL",
240            "FH_INSTRUMENT",
241            "FH_STRIKE_PRICE",
242            "FH_LAST_TRADED_PRICE",
243            "TIMESTAMP",
244        ]
245    if response_type == "json":
246        data_json_ret = []
247        for record in data_json:
248            for column in columns_list:
249                record.pop(column, None)
250            data_json_ret.append(record)
251        return data_json_ret
252
253    return (
254        pd.json_normalize(data_json)
255        .drop(columns=columns_list)
256        .rename(
257            columns={
258                "FH_OPENING_PRICE": "Open Price",
259                "FH_TRADE_HIGH_PRICE": "High Price",
260                "FH_CLOSING_PRICE": "Close Price",
261                "FH_TRADE_LOW_PRICE": "Low Price",
262                "FH_CHANGE_IN_OI": "Change in OI",
263                "FH_EXPIRY_DT": "Expiry Date",
264                "FH_TIMESTAMP": "Date",
265            }
266        )
267    )
268
269
270def derivaties_options(
271    data_json: str,
272    response_type: str = "panda_df",
273    columns_drop_list=None,
274):
275    """
276        Format historical options data
277
278    Args:
279        data_json (object): data in json format.
280        response_type (str, optional): response_type. Defaults to "panda_df".
281        columns_drop_list (list, optional): custom columns drop list. Defaults to None.
282
283    Returns:
284            json: format data in json
285        or
286            dataframe: format data in panda df
287    """
288    if data_json:
289        data_json = data_json['data']
290    if columns_drop_list:
291        columns_list = columns_drop_list
292    else:
293        columns_list = [
294            "_id",
295            "FH_MARKET_LOT",
296            "FH_MARKET_TYPE",
297            "FH_OPTION_TYPE",
298            "FH_SYMBOL",
299            "FH_INSTRUMENT",
300            "FH_STRIKE_PRICE",
301            "FH_LAST_TRADED_PRICE",
302            "TIMESTAMP",
303        ]
304
305    if response_type == "json":
306        data_json_ret = []
307        for record in data_json:
308            for column in columns_list:
309                record.pop(column, None)
310            data_json_ret.append(record)
311        return data_json_ret
312
313    return (
314        pd.json_normalize(data_json)
315        .drop(columns=columns_list)
316        .rename(
317            columns={
318                "FH_OPENING_PRICE": "Open Price",
319                "FH_TRADE_HIGH_PRICE": "High Price",
320                "FH_CLOSING_PRICE": "Close Price",
321                "FH_TRADE_LOW_PRICE": "Low Price",
322                "FH_CHANGE_IN_OI": "Change in OI",
323                "FH_EXPIRY_DT": "Expiry Date",
324                "FH_TIMESTAMP": "Date",
325            }
326        )
327    )

def price(result): View Source

 9def price(result):
10    """
11    Args:
12        result (Pandas DataFrame): result
13
14    Returns:
15        Pandas DataFrame: df containing data in specific format
16    """
17
18    columns_required = [
19        "CH_TIMESTAMP",
20        "CH_OPENING_PRICE",
21        "CH_TRADE_HIGH_PRICE",
22        "CH_TRADE_LOW_PRICE",
23        "CH_CLOSING_PRICE",
24        "CH_PREVIOUS_CLS_PRICE",
25        "CH_LAST_TRADED_PRICE",
26        "CH_TOT_TRADED_QTY",
27        "CH_TOT_TRADED_VAL",
28        "CH_52WEEK_HIGH_PRICE",
29        "CH_52WEEK_LOW_PRICE",
30        "VWAP",
31        "COP_DELIV_QTY",
32        "COP_DELIV_PERC",
33        "CH_SERIES",
34    ]
35
36    try:
37        result = result[columns_required]
38    except:  # pylint: disable=W0702
39        return result
40    result = result.set_axis(
41        [
42            "Date",
43            "Open Price",
44            "High Price",
45            "Low Price",
46            "Close Price",
47            "Prev Close Price",
48            "Last Traded Price",
49            "Total Traded Quantity",
50            "Total Traded Value",
51            "52 Week High Price",
52            "52 Week Low Price",
53            "VWAP",
54            "Deliverable Volume",
55            "Deliverable Percent",
56            "Series",
57        ],
58        axis=1,
59    )
60
61    result["Date"] = pd.to_datetime(result["Date"])
62    result = result.sort_values("Date", ascending=True)
63    result.reset_index(drop=True, inplace=True)
64    return result

Args: result (Pandas DataFrame): result

Returns: Pandas DataFrame: df containing data in specific format

def indices( data_json, columns_drop_list: list = None, columns_rename_map: map = None): View Source

 67def indices(
 68    data_json,
 69    columns_drop_list: list = None,
 70    columns_rename_map: map = None,
 71):
 72    """
 73    Args:
 74        data_json (json):  data in json format
 75    Returns:
 76        Pandas DataFrame: df with indexCloseOnlineRecords and indexTurnoverRecords
 77    """
 78    if columns_drop_list:
 79        columns_list = columns_drop_list
 80    else:
 81        columns_list = ["_id", "EOD_INDEX_NAME", "TIMESTAMP"]
 82
 83    if columns_rename_map:
 84        columns_rename = columns_rename_map
 85    else:
 86        columns_rename = {
 87            "EOD_OPEN_INDEX_VAL": "Open Price",
 88            "EOD_HIGH_INDEX_VAL": "High Price",
 89            "EOD_CLOSE_INDEX_VAL": "Close Price",
 90            "EOD_LOW_INDEX_VAL": "Low Price",
 91            "EOD_TIMESTAMP": "Date",
 92        }
 93    data_close_df = (
 94        pd.DataFrame(data_json["data"]["indexCloseOnlineRecords"])
 95        .drop(columns=columns_list)
 96        .rename(columns=columns_rename)
 97    )
 98
 99    ## Mismatch values
100    # data_turnover_df = (
101    #     pd.DataFrame(d["data"]["indexTurnoverRecords"])
102    #     .drop(columns=["_id", "HIT_INDEX_NAME_UPPER", "HIT_TIMESTAMP"])
103    #     .rename(
104    #         columns={
105    #             "HIT_TRADED_QTY": "Total Traded Quantity",
106    #             "HIT_TURN_OVER": "Total Traded Value",
107    #             "TIMESTAMP": "Date",
108    #         }
109    #     )
110    # )
111    return data_close_df
112    # return pd.merge(data_close_df, data_turnover_df, on="Date", how="inner")

Args: data_json (json): data in json format Returns: Pandas DataFrame: df with indexCloseOnlineRecords and indexTurnoverRecords

def option_chain(data_json: str, response_type: str): View Source

115def option_chain(
116    data_json: str,
117    response_type: str,
118):
119    """_summary_
120
121    Args:
122        data_json (str): _description_
123        response_type (str): _description_
124
125    Returns:
126        _type_: _description_
127    """
128    if response_type == "json":
129        data_json_ret = []
130        for record in data_json:
131            if "PE" in record:
132                record["PE"].pop("strikePrice", None)
133                record["PE"].pop("expiryDate", None)
134                record["PE"].pop("underlying", None)
135                record["PE"].pop("identifier", None)
136            if "CE" in record:
137                record["CE"].pop("strikePrice", None)
138                record["CE"].pop("expiryDate", None)
139                record["CE"].pop("underlying", None)
140                record["CE"].pop("identifier", None)
141            data_json_ret.append(record)
142        return data_json_ret
143
144    return (
145        pd.json_normalize(data_json)
146        .sort_values(by=["expiryDate", "strikePrice"], ascending=True)
147        .drop(
148            columns=[
149                "PE.strikePrice",
150                "PE.expiryDate",
151                "PE.identifier",
152                "CE.strikePrice",
153                "CE.expiryDate",
154                "CE.identifier",
155            ]
156        )
157    )

_summary_

Args: data_json (str): _description_ response_type (str): _description_

Returns: _type_: _description_

def get_vix( data_json: object, response_type: str = 'panda_df', columns_drop_list: list = None): View Source

160def get_vix(
161    data_json: object,
162    response_type: str = "panda_df",
163    columns_drop_list: list = None,
164):
165    """
166        Format Vix data
167    Args:
168        data_json (object): data in json format.
169        response_type (str, optional): response_type. Defaults to "panda_df".
170        columns_drop_list (list, optional): custom columns drop list. Defaults to None.
171    Returns:
172        _type_: _description_
173    """
174    data_json = data_json["data"]
175    if columns_drop_list:
176        columns_list = columns_drop_list
177    else:
178        columns_list = [
179            "_id",
180            "TIMESTAMP",
181            "createdAt",
182            "updatedAt",
183            "__v",
184            "ALTERNATE_INDEX_NAME",
185            "EOD_INDEX_NAME",
186            "EOD_PREV_CLOSE",
187            "VIX_PTS_CHG",
188            "VIX_PERC_CHG",
189        ]
190    if response_type == "json":
191        data_json_ret = []
192        for record in data_json:
193            for column in columns_list:
194                record.pop(column, None)
195
196            data_json_ret.append(record)
197        return data_json_ret
198
199    return (
200        pd.json_normalize(data_json)
201        .drop(columns=columns_list)
202        .rename(
203            columns={
204                "EOD_OPEN_INDEX_VAL": "Open Price",
205                "EOD_HIGH_INDEX_VAL": "High Price",
206                "EOD_CLOSE_INDEX_VAL": "Close Price",
207                "EOD_LOW_INDEX_VAL": "Low Price",
208                "EOD_TIMESTAMP": "Date",
209            }
210        )
211    )

Format Vix data Args: data_json (object): data in json format. response_type (str, optional): response_type. Defaults to "panda_df". columns_drop_list (list, optional): custom columns drop list. Defaults to None. Returns: _type_: _description_

def derivatives_futures( data_json: str, response_type: str = 'panda_df', columns_drop_list=None): View Source

214def derivatives_futures(
215    data_json: str,
216    response_type: str = "panda_df",
217    columns_drop_list=None,
218):
219    """
220        Format futures data
221
222    Args:
223        data_json (object): data in json format.
224        response_type (str, optional): response_type. Defaults to "panda_df".
225        columns_drop_list (list, optional): custom columns drop list. Defaults to None.
226
227    Returns:
228            json: format data in json
229        or
230            dataframe: format data in panda df
231    """
232    if columns_drop_list:
233        columns_list = columns_drop_list
234    else:
235        columns_list = [
236            "_id",
237            "FH_MARKET_LOT",
238            "FH_MARKET_TYPE",
239            "FH_OPTION_TYPE",
240            "FH_SYMBOL",
241            "FH_INSTRUMENT",
242            "FH_STRIKE_PRICE",
243            "FH_LAST_TRADED_PRICE",
244            "TIMESTAMP",
245        ]
246    if response_type == "json":
247        data_json_ret = []
248        for record in data_json:
249            for column in columns_list:
250                record.pop(column, None)
251            data_json_ret.append(record)
252        return data_json_ret
253
254    return (
255        pd.json_normalize(data_json)
256        .drop(columns=columns_list)
257        .rename(
258            columns={
259                "FH_OPENING_PRICE": "Open Price",
260                "FH_TRADE_HIGH_PRICE": "High Price",
261                "FH_CLOSING_PRICE": "Close Price",
262                "FH_TRADE_LOW_PRICE": "Low Price",
263                "FH_CHANGE_IN_OI": "Change in OI",
264                "FH_EXPIRY_DT": "Expiry Date",
265                "FH_TIMESTAMP": "Date",
266            }
267        )
268    )

Format futures data

Args: data_json (object): data in json format. response_type (str, optional): response_type. Defaults to "panda_df". columns_drop_list (list, optional): custom columns drop list. Defaults to None.

Returns: json: format data in json or dataframe: format data in panda df

def derivaties_options( data_json: str, response_type: str = 'panda_df', columns_drop_list=None): View Source

271def derivaties_options(
272    data_json: str,
273    response_type: str = "panda_df",
274    columns_drop_list=None,
275):
276    """
277        Format historical options data
278
279    Args:
280        data_json (object): data in json format.
281        response_type (str, optional): response_type. Defaults to "panda_df".
282        columns_drop_list (list, optional): custom columns drop list. Defaults to None.
283
284    Returns:
285            json: format data in json
286        or
287            dataframe: format data in panda df
288    """
289    if data_json:
290        data_json = data_json['data']
291    if columns_drop_list:
292        columns_list = columns_drop_list
293    else:
294        columns_list = [
295            "_id",
296            "FH_MARKET_LOT",
297            "FH_MARKET_TYPE",
298            "FH_OPTION_TYPE",
299            "FH_SYMBOL",
300            "FH_INSTRUMENT",
301            "FH_STRIKE_PRICE",
302            "FH_LAST_TRADED_PRICE",
303            "TIMESTAMP",
304        ]
305
306    if response_type == "json":
307        data_json_ret = []
308        for record in data_json:
309            for column in columns_list:
310                record.pop(column, None)
311            data_json_ret.append(record)
312        return data_json_ret
313
314    return (
315        pd.json_normalize(data_json)
316        .drop(columns=columns_list)
317        .rename(
318            columns={
319                "FH_OPENING_PRICE": "Open Price",
320                "FH_TRADE_HIGH_PRICE": "High Price",
321                "FH_CLOSING_PRICE": "Close Price",
322                "FH_TRADE_LOW_PRICE": "Low Price",
323                "FH_CHANGE_IN_OI": "Change in OI",
324                "FH_EXPIRY_DT": "Expiry Date",
325                "FH_TIMESTAMP": "Date",
326            }
327        )
328    )

Format historical options data

Args: data_json (object): data in json format. response_type (str, optional): response_type. Defaults to "panda_df". columns_drop_list (list, optional): custom columns drop list. Defaults to None.

Returns: json: format data in json or dataframe: format data in panda df