Skip to content

Commit 4f969c6

Browse files
JerBoumaJerBouma
committed
Fix some errors add more robustness
1 parent 9282f7d commit 4f969c6

3 files changed

Lines changed: 160119 additions & 160000 deletions

File tree

‎.github/workflows/database_update.yml‎

Lines changed: 182 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,145 @@ jobs:
1616
- name: setup python
1717
uses: actions/setup-python@v4
1818
with:
19-
python-version: '3.10'
19+
python-version: '3.13'
2020
- run: pip install "pandas[excel]" financedatabase
2121
- name: Add New Tickers and Update Old Ones
2222
uses: jannekem/run-python-script-action@v1
2323
with:
2424
script: |
2525
import numpy as np
2626
import pandas as pd
27+
from typing import Optional
28+
29+
# Market cap thresholds in USD
30+
MARKET_CAP_THRESHOLDS = {
31+
'Mega Cap': 200_000_000_000,
32+
'Large Cap': 10_000_000_000,
33+
'Mid Cap': 2_000_000_000,
34+
'Small Cap': 300_000_000,
35+
'Micro Cap': 50_000_000,
36+
'Nano Cap': 0,
37+
}
38+
39+
def calculate_market_cap(value: Optional[float]) -> Optional[str]:
40+
"""Categorize a market capitalization value into a named tier.
41+
42+
Args:
43+
value: Market capitalization in USD, or None/NaN.
44+
45+
Returns:
46+
Market cap tier label, or np.nan if value is missing/zero.
47+
"""
48+
if pd.isna(value) or not value:
49+
return np.nan
50+
for label, threshold in MARKET_CAP_THRESHOLDS.items():
51+
if float(value) >= threshold:
52+
return label
53+
return np.nan
54+
55+
56+
def lookup_industry(row_industry: str, fd_industries: pd.DataFrame) -> Optional[str]:
57+
"""Map an exchange industry label to the FinanceDatabase equivalent.
58+
59+
Args:
60+
row_industry: Raw industry string from the exchange data.
61+
fd_industries: Lookup DataFrame with FinanceDatabase industry names.
62+
63+
Returns:
64+
Mapped industry string, or np.nan if not found.
65+
"""
66+
try:
67+
result = fd_industries.loc[row_industry].iloc[0]
68+
return result.iloc[0] if isinstance(result, pd.Series) else result
69+
except KeyError:
70+
return np.nan
71+
72+
73+
def lookup_industry_group(industry: str, equities: pd.DataFrame) -> Optional[str]:
74+
"""Infer the most common industry group for a given industry.
75+
76+
Args:
77+
industry: FinanceDatabase industry string.
78+
equities: The equities reference DataFrame.
79+
80+
Returns:
81+
Most frequent industry_group value, or np.nan if not found.
82+
"""
83+
if pd.isna(industry):
84+
return np.nan
85+
subset = equities[equities['industry'] == industry]
86+
return subset['industry_group'].mode()[0] if not subset.empty else np.nan
87+
88+
89+
def lookup_sector(industry: str, industry_group: str, equities: pd.DataFrame) -> Optional[str]:
90+
"""Infer the most common sector for a given industry and industry group.
91+
92+
Args:
93+
industry: FinanceDatabase industry string.
94+
industry_group: FinanceDatabase industry_group string.
95+
equities: The equities reference DataFrame.
96+
97+
Returns:
98+
Most frequent sector value, or np.nan if not found.
99+
"""
100+
if pd.isna(industry) or pd.isna(industry_group):
101+
return np.nan
102+
subset = equities[
103+
(equities['industry_group'] == industry_group) &
104+
(equities['industry'] == industry)
105+
]
106+
return subset['sector'].mode()[0] if not subset.empty else np.nan
107+
108+
109+
def build_new_ticker(
110+
index: str,
111+
row: pd.Series,
112+
fd_industries: pd.DataFrame,
113+
equities: pd.DataFrame,
114+
market_cap: Optional[str],
115+
) -> dict:
116+
"""Build a new ticker entry for the equities database.
117+
118+
Args:
119+
index: Ticker symbol.
120+
row: Raw row from the exchange data.
121+
fd_industries: Lookup DataFrame for industries.
122+
equities: The equities reference DataFrame.
123+
market_cap: Pre-calculated market cap tier.
124+
125+
Returns:
126+
Dictionary with all required equity fields.
127+
"""
128+
industry = lookup_industry(row['industry'], fd_industries)
129+
industry_group = lookup_industry_group(industry, equities)
130+
sector = lookup_sector(industry, industry_group, equities)
131+
132+
return {
133+
'name': row['name'],
134+
'summary': np.nan,
135+
'currency': 'USD',
136+
'industry': industry,
137+
'industry_group': industry_group,
138+
'sector': sector,
139+
'exchange': row['exchange'],
140+
'market': row['market'],
141+
'country': row['country'],
142+
'state': np.nan,
143+
'city': np.nan,
144+
'zipcode': np.nan,
145+
'website': np.nan,
146+
'market_cap': market_cap,
147+
'isin': np.nan,
148+
'cusip': np.nan,
149+
'figi': np.nan,
150+
'composite_figi': np.nan,
151+
'shareclass_figi': np.nan,
152+
}
153+
154+
155+
# ---------------------------------------------------------------------------
156+
# Data ingestion
157+
# ---------------------------------------------------------------------------
27158
28159
# Collect NASDAQ data
29160
nasdaq = pd.read_json("https://raw.githubusercontent.com/rreichel3/US-Stock-Symbols/main/nasdaq/nasdaq_full_tickers.json")
@@ -37,133 +168,83 @@ jobs:
37168
nyse['exchange'] = 'ASE'
38169
nyse['market'] = 'NYSE MKT'
39170
40-
# Collect AMEX data, since it got acquired this is now the same exchange/market as NYSE
171+
# Collect AMEX data (acquired by NYSE, same exchange/market)
41172
amex = pd.read_json("https://raw.githubusercontent.com/rreichel3/US-Stock-Symbols/main/amex/amex_full_tickers.json")
42173
amex = amex.set_index('symbol')
43174
amex['exchange'] = 'ASE'
44175
amex['market'] = 'NYSE MKT'
45176
46-
# Combine the datasets
177+
# Combine all exchange datasets
47178
exchange_data = pd.concat([nasdaq, nyse, amex])
48179
49-
# Obtain the categories from the FinanceDatabase for conversion
180+
# ---------------------------------------------------------------------------
181+
# Reference data
182+
# ---------------------------------------------------------------------------
183+
50184
fd_categories_path = 'compression/categories/github_exchange_categories.xlsx'
51-
fd_sectors = pd.read_excel(fd_categories_path, sheet_name='sector', index_col=1)
185+
fd_sectors = pd.read_excel(fd_categories_path, sheet_name='sector', index_col=1)
52186
fd_industry_groups = pd.read_excel(fd_categories_path, sheet_name='industry_group', index_col=1)
53-
fd_industries = pd.read_excel(fd_categories_path, sheet_name='industry', index_col=1)
187+
fd_industries = pd.read_excel(fd_categories_path, sheet_name='industry', index_col=1)
54188
55189
# Read the equities database
56190
equities = pd.read_csv('database/equities.csv', index_col=0)
57-
ticker_dict = {}
58191
59-
# Loop over the exchange dataset and create a new object that will be added to the database
192+
# ---------------------------------------------------------------------------
193+
# Main processing loop
194+
# ---------------------------------------------------------------------------
195+
196+
ticker_dict: dict = {}
197+
60198
for index, row in exchange_data.iterrows():
61-
if row['marketCap']:
62-
market_cap_value = float(row['marketCap'])
63-
64-
if market_cap_value >= 200_000_000_000:
65-
market_cap = 'Mega Cap'
66-
elif market_cap_value >= 10_000_000_000 and market_cap_value < 200_000_000_000:
67-
market_cap= 'Large Cap'
68-
elif market_cap_value >= 2_000_000_000 and market_cap_value < 10_000_000_000:
69-
market_cap = 'Mid Cap'
70-
elif market_cap_value >= 300_000_000 and market_cap_value < 2_000_000_000:
71-
market_cap = 'Small Cap'
72-
elif market_cap_value >= 50_000_000 and market_cap_value < 300_000_000:
73-
market_cap = 'Micro Cap'
74-
else:
75-
market_cap = 'Nano Cap'
76-
else:
77-
market_cap = np.nan
78-
199+
market_cap = calculate_market_cap(row.get('marketCap'))
200+
79201
try:
80-
# Checks if ticker exists, if yes, continue
81202
fd_data = equities.loc[index]
82-
83-
if len(fd_data) == 1 and fd_data['market_cap'] != market_cap and market_cap == market_cap:
84-
ticker_dict[index] = {'symbol': index}
85-
for column, value in fd_data.items():
86-
if column == 'market_cap':
87-
ticker_dict[index][column] = market_cap
88-
else:
89-
ticker_dict[index][column] = value
203+
204+
# Update market_cap only when it has changed and the new value is valid
205+
if len(fd_data) == 0 and fd_data['market_cap'] != market_cap and pd.notna(market_cap):
206+
ticker_dict[index] = {'symbol': index, **fd_data.to_dict(), 'market_cap': market_cap}
90207
continue
208+
91209
except KeyError:
92-
if index != index:
93-
# Specific case where the ticker is NA which is recognized
94-
# as a NaN instead meaning it will continuously be added
95-
index = "NA"
96-
97-
ticker_dict[index] = {}
98-
99-
ticker_dict[index]['name'] = row['name']
100-
ticker_dict[index]['summary'] = np.nan
101-
ticker_dict[index]['currency'] = "USD"
102-
103-
try:
104-
industry = fd_industries.loc[row['industry']].iloc[0]
105-
106-
if isinstance(industry, pd.Series):
107-
industry = industry[0]
108-
109-
ticker_dict[index]['industry'] = industry
110-
except KeyError:
111-
ticker_dict[index]['industry'] = np.nan
112-
113-
try:
114-
industry_divison = equities[equities['industry'] == ticker_dict[index]['industry']]
115-
industry_group = industry_divison['industry_group'].mode()[0]
116-
117-
ticker_dict[index]['industry_group'] = industry_group
118-
except KeyError:
119-
ticker_dict[index]['industry_group'] = np.nan
120-
121-
try:
122-
sector_division = equities[(equities['industry_group'] == ticker_dict[index]['industry_group']) & (equities['industry'] == ticker_dict[index]['industry'])]
123-
sector = sector_division['sector'].mode()[0]
210+
# "NA" is parsed as NaN by pandas; normalise the index back to a string
211+
if pd.isna(index):
212+
index = "NA"
213+
214+
ticker_dict[index] = build_new_ticker(index, row, fd_industries, equities, market_cap)
215+
216+
# ---------------------------------------------------------------------------
217+
# Merge results back into equities
218+
# ---------------------------------------------------------------------------
124219
125-
ticker_dict[index]['sector'] = sector
126-
except Exception:
127-
ticker_dict[index]['sector'] = np.nan
128-
129-
ticker_dict[index]['exchange'] = row['exchange']
130-
ticker_dict[index]['market'] = row['market']
131-
ticker_dict[index]['country'] = row['country']
132-
ticker_dict[index]['state'] = np.nan
133-
ticker_dict[index]['city'] = np.nan
134-
ticker_dict[index]['zipcode'] = np.nan
135-
ticker_dict[index]['website'] = np.nan
136-
ticker_dict[index]['market_cap'] = market_cap
137-
ticker_dict[index]['isin'] = np.nan
138-
ticker_dict[index]['cusip'] = np.nan
139-
ticker_dict[index]['figi'] = np.nan
140-
ticker_dict[index]['composite_figi'] = np.nan
141-
ticker_dict[index]['shareclass_figi'] = np.nan
142-
143-
# Create a DataFrame out of the created dictionary
144220
updated_companies = pd.DataFrame.from_dict(ticker_dict, orient='index')
145221
updated_companies.index.name = 'symbol'
146222
223+
# Drop accidental 'symbol' column that may surface from existing-row updates
224+
updated_companies = updated_companies.drop(columns=['symbol'], errors='ignore')
225+
147226
print(f"There are {len(updated_companies)} new updates!")
148227
149228
if not updated_companies.empty:
150-
# Loop over all acquired values and update data
151-
for index, values in updated_companies.iterrows():
152-
try:
153-
equities.loc[index] = updated_companies.loc[index]
154-
except KeyError:
155-
equities = pd.concat([equities, values])
156-
157-
# Sort the index
158-
equities = equities.sort_index()
229+
# Update existing rows in-place
230+
existing_indices = updated_companies.index.intersection(equities.index)
231+
if not existing_indices.empty:
232+
equities.update(updated_companies.loc[existing_indices])
233+
234+
# Append completely new tickers
235+
new_indices = updated_companies.index.difference(equities.index)
236+
if not new_indices.empty:
237+
equities = pd.concat([equities, updated_companies.loc[new_indices]])
238+
239+
equities = equities[~equities.index.duplicated(keep='first')]
240+
equities = equities[equities.index.notna()]
159241
160-
# Drop NaN values in the index
161-
new_index = equities.index.dropna()
242+
equities = (
243+
equities
244+
.sort_index()
245+
.loc[equities.index.notna()] # drop NaN index entries
246+
)
162247
163-
# Update the equities DataFrame
164-
equities = equities.loc[new_index]
165-
166-
# Send to CSV
167248
equities.to_csv('database/equities.csv')
168249
- name: Commit files and log
169250
run: |

‎.github/workflows/linting.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
run: poetry run black --diff --check .
4444

4545
- name: Run ruff
46-
run: poetry run ruff financedatabase
46+
run: poetry run ruff check financedatabase
4747

4848
- name: Run pylint
4949
run: poetry run pylint financedatabase

0 commit comments

Comments
 (0)