save_xml() for the interface

epogrebnyak · epogrebnyak · commit ecdeed833a85 · 2022-03-12T10:31:07.000Z
diff --git a/README.md b/README.md
@@ -4,53 +4,60 @@
 
 Data source page: ["Daily Treasury Par Yield Curve Rates"](https://home.treasury.gov/resource-center/data-chart-center/interest-rates/TextView?type=daily_treasury_yield_curve&field_tdr_date_value=2022) 
 
-## Large dataset
+## Read existing dataset 
 
-CSV file (1990-present):
-
- - [daily yields](https://raw.githubusercontent.com/epogrebnyak/data-ust/master/ust.csv)
+CSV file (1990-present): [daily UST yields](https://raw.githubusercontent.com/epogrebnyak/data-ust/master/ust.csv)
  
-## How to use
+```
+url = "https://raw.githubusercontent.com/epogrebnyak/data-ust/master/ust.csv"
+df = pd.read_csv(df, parse_dates=["date"]).set_index("date")
+```
 
-Build a local file (`rates.csv`) with US Treasuries yields by maturity: 
+## Download current dataset
+
+Build a local CSV file with US Treasuries yields by maturity: 
 
 ```python
 import os
+
 import pandas as pd
-from ust import save, force_save, read_rates
+
+from ust import save_xml, read_rates, available_years
 
 # save UST yield rates to local folder for selected years
-for year in [2020, 2021, 2022]:
-    save(year, folder="./xml")
+for year in available_years():
+    save_xml(year, folder="./xml")
 
-# force update last year (overwrites existing file)
-force_save(2022, folder="./xml")
+# run later - force update last year (overwrites existing file)
+save_xml(2022, folder="./xml", overwrite=True)
 
-# read as pandas dataframe
+# read UST yield rates as pandas dataframe
 df = read_rates(start_year=2020, end_year=2022, folder="./xml")
+print(df)
 
 # save as single CSV file
 df.to_csv("rates.csv")
 
 # read back later
 df = pd.read_csv("rates.csv", parse_dates=["date"]).set_index("date")
 
+
 """
-            BC_1MONTH  BC_3MONTH  BC_6MONTH  ...  BC_20YEAR  BC_30YEAR  BC_30YEARDISPLAY
-date                                         ...                                        
+            BC_1MONTH  BC_3MONTH  BC_6MONTH  BC_1YEAR  ...  BC_10YEAR  BC_20YEAR  BC_30YEAR  BC_30YEARDISPLAY
+date                                                   ...                                                   
 2020-01-02       1.53       1.54       1.57      1.56  ...       1.88       2.19       2.33              2.33
 2020-01-03       1.52       1.52       1.55      1.55  ...       1.80       2.11       2.26              2.26
 2020-01-06       1.54       1.56       1.56      1.54  ...       1.81       2.13       2.28              2.28
 2020-01-07       1.52       1.54       1.56      1.53  ...       1.83       2.16       2.31              2.31
 2020-01-08       1.50       1.54       1.56      1.55  ...       1.87       2.21       2.35              2.35
 ...               ...        ...        ...       ...  ...        ...        ...        ...               ...
-2022-03-03       0.19       0.38       0.69      1.08  ...       1.86       2.32       2.24              2.24
-2022-03-04       0.15       0.34       0.69      1.05  ...       1.74       2.23       2.16              2.16
 2022-03-07       0.17       0.38       0.75      1.07  ...       1.78       2.29       2.19              2.19
 2022-03-08       0.16       0.36       0.72      1.12  ...       1.86       2.34       2.24              2.24
 2022-03-09       0.18       0.38       0.75      1.15  ...       1.94       2.38       2.29              2.29
-              2.29
-"""
+2022-03-10       0.19       0.39       0.75      1.19  ...       1.98       2.45       2.38              2.38
+2022-03-11       0.17       0.40       0.78      1.22  ...       2.00       2.45       2.36              2.36
+
+[550 rows x 12 columns]
 ```
 
 ## Sample image
diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -0,0 +1,3 @@
+black
+pyright
+pytest
diff --git a/runner.py b/runner.py
@@ -1,38 +1,20 @@
 import os
-
 import pandas as pd
-
-from ust import force_save, read_rates, save
+from ust import save_xml, read_rates, available_years
 
 # save UST yield rates to local folder for selected years
-for year in [2020, 2021, 2022]:
-    save(year, folder="./xml")
+for year in available_years():
+    save_xml(year, folder="./xml")
 
-# force update last year (overwrites existing file)
-force_save(2022, folder="./xml")
+# run later - force update last year (overwrites existing file)
+save_xml(2022, folder="./xml", overwrite=True)
 
 # read UST yield rates as pandas dataframe
-df = read_rates(start_year=2021, end_year=2022, folder="./xml")
+df = read_rates(start_year=2020, end_year=2022, folder="./xml")
+print(df)
 
 # save as single CSV file
 df.to_csv("rates.csv")
 
 # read back later
 df = pd.read_csv("rates.csv", parse_dates=["date"]).set_index("date")
-
-
-"""
-            BC_1MONTH  BC_3MONTH  BC_6MONTH  ...  BC_20YEAR  BC_30YEAR  BC_30YEARDISPLAY
-date                                         ...                                        
-2021-01-04       0.09       0.09       0.09  ...       1.46       1.66              1.66
-2021-01-05       0.08       0.09       0.09  ...       1.49       1.70              1.70
-2021-01-06       0.09       0.09       0.09  ...       1.60       1.81              1.81
-2021-01-07       0.09       0.09       0.09  ...       1.64       1.85              1.85
-2021-01-08       0.08       0.08       0.09  ...       1.67       1.87              1.87
-...               ...        ...        ...  ...        ...        ...               ...
-2022-03-02       0.13       0.34       0.68  ...       2.32       2.24              2.24
-2022-03-03       0.19       0.38       0.69  ...       2.32       2.24              2.24
-2022-03-04       0.15       0.34       0.69  ...       2.23       2.16              2.16
-2022-03-07       0.17       0.38       0.75  ...       2.29       2.19              2.19
-2022-03-08       0.16       0.36       0.72  ...       2.34       2.24              2.24
-"""
diff --git a/ust.csv b/ust.csv
@@ -8052,3 +8052,5 @@ date,BC_1MONTH,BC_3MONTH,BC_6MONTH,BC_1YEAR,BC_2YEAR,BC_3YEAR,BC_5YEAR,BC_7YEAR,
 2022-03-07,0.17,0.38,0.75,1.07,1.55,1.68,1.71,1.77,1.78,2.29,2.19,2.19
 2022-03-08,0.16,0.36,0.72,1.12,1.63,1.78,1.8,1.85,1.86,2.34,2.24,2.24
 2022-03-09,0.18,0.38,0.75,1.15,1.68,1.83,1.87,1.92,1.94,2.38,2.29,2.29
+2022-03-10,0.19,0.39,0.75,1.19,1.72,1.88,1.92,1.98,1.98,2.45,2.38,2.38
+2022-03-11,0.17,0.4,0.78,1.22,1.75,1.91,1.96,2.01,2.0,2.45,2.36,2.36
diff --git a/ust.png b/ust.png
diff --git a/ust.py b/ust.py
@@ -134,7 +134,7 @@ def data(self):
         return soup.find_all("content")
 
 
-def get_date(string):
+def get_date(string) -> str:
     dt = datetime.strptime(string, "%Y-%m-%dT%H:%M:%S")
     return dt.strftime("%Y-%m-%d")
 
@@ -145,7 +145,8 @@ def yield_datapoints_from_string(xml_content) -> Iterable[dict]:
     data = soup.find_all("content")
     for datum in data:
         cur_dict = dict((key, elem(datum, key)) for key in BC_KEYS)
-        cur_dict["date"] = get_date(datum.find("NEW_DATE").text)
+        # ignore type check with pyright below
+        cur_dict["date"] = get_date(datum.find("NEW_DATE").text)  # type: ignore
         yield cur_dict
 
 
@@ -194,7 +195,7 @@ def year_now():
 
 
 def available_years():
-    return range(1990, year_now() + 1)
+    return list(range(1990, year_now() + 1))
 
 
 def from_years(years: list, folder: str):
@@ -218,13 +219,20 @@ def years(start_year, end_year):
     return range(start_year, end_year + 1)
 
 
-def force_save(year, folder=default_folder()):
+def save_xml(year, folder, overwrite=False):
+    if overwrite:
+        force_save(year, folder)
+    else:
+        soft_save(year, folder)
+
+
+def force_save(year, folder):
     r = Rates(year, folder)
     r.save_local()
     print("Updated", r.path)
 
 
-def save(year, folder=default_folder()):
+def soft_save(year, folder):
     r = Rates(year, folder)
     if not r.exists():
         r.save_local()
@@ -233,18 +241,12 @@ def save(year, folder=default_folder()):
         print("No action taken - file already exists", r.path)
 
 
-
-def save_rates(start_year, end_year, folder=default_folder()):
+def save_rates(start_year, end_year, folder):
     for year in years(start_year, end_year):
-        r = Rates(year, folder)
-        if not r.exists():
-            r.save_local()
-            print("Saved data for year", year)
-        else:
-            print("No action taken - file already exists", r.path)
+        soft_save(year, folder)
 
 
-def read_rates(start_year, end_year, folder=default_folder()):
+def read_rates(start_year, end_year, folder):
     dfs = []
     for year in years(start_year, end_year):
         r = Rates(year, folder)
@@ -254,7 +256,7 @@ def read_rates(start_year, end_year, folder=default_folder()):
 
 def draw(folder=default_folder()):
     current_year = year_now()
-    force_save(current_year)
+    save_xml(current_year, folder, overwrite=True)
     df = read_rates(1990, current_year, folder)
     df.to_csv("ust.csv")
     make_chart(df, "ust.png")
diff --git a/xml/2022.xml b/xml/2022.xml
