Skip to content
GitLab
Commit cd98bc4e authored by Lee KIMBER's avatar Lee KIMBER
Browse files

Fixes configs and Aegon_Scraper.py to working state.

parent 7e3afd83
Expand all Hide whitespace changes
Inline Side-by-side
Aegon_Pension.py
View file @ cd98bc4e
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
from urllib.parse import quote_plus, urlencode
from datetime import date
from pymongo import MongoClient
import json
import importlib
import requests
import traceback
from Aegon_Scraper import main
my_module = importlib.import_module(
os.path.splitext((os.path.basename(__file__)))[0] + "_config"
)
debug = my_module.debug
uri_template = my_module.uri_template
payload = my_module.payload
headers = my_module.headers
scheme = my_module.scheme
database = my_module.database
mongoServer = my_module.mongoServer
mongoPort = my_module.mongoPort
PER_PAGE = my_module.PER_PAGE
MAX_ROWS = my_module.MAX_ROWS
ae_label = "aegonportal"
enable_sedol = True
connection = MongoClient(mongoServer, mongoPort)
db = connection[database]
scheme = "Aegon_Pension"
def get_logger():
# Set up logging
file_path = os.path.basename(sys.argv[0])
filename = file_path.replace(".py", "")
# logfile = "/tmp/" + filename + "_pension.log"
logfile = filename + "_pension.log"
logging.basicConfig(
filename=logfile,
format="%(asctime)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO,
)
logger = logging.getLogger(__name__)
logger.info("#####################")
logger.info("Starting %s", file_path)
return logger
def get_timestamp():
today = date.today().toordinal()
lastWeek = today - 7
sunday = lastWeek - (lastWeek % 7)
friday = sunday + 5
timestamp = date.fromordinal(friday)
return timestamp
timestamp = get_timestamp()
def get_dbfunds(funds_data):
dbfunds = {}
for doc in funds_data.find({}):
fundname = doc["fundname"]
last_data = doc["prices"][-1]
price_string = (last_data["sell_price"], last_data["buy_price"])
dbfunds[fundname] = price_string
return dbfunds
def get_webfunds(ae_label, sedol_search):
webfunds = dict()
total = 0
page_number = 1
while True:
pl = payload.copy()
pl["PageNo"] = page_number
pl["PageSize"] = PER_PAGE
pl["RowIDs"] = ",".join(
[
str(i)
for i in range(
PER_PAGE * (page_number - 1) + 1, page_number * PER_PAGE + 1
)
]
)
pl["ProjectName"] = ae_label
pl["EnableSedolSearch"] = sedol_search
url = uri_template.format(quote_plus(json.dumps(pl)))
try:
page = requests.get(url)
data = json.loads(page.text)
except:
logger = get_logger()
logger.error(traceback.format_exc())
logging.error("Error downloading page at: " + url)
sys.exit(1)
else:
data = json.loads(data)
data["TotalRows"] = data.get("TotalRows") or MAX_ROWS
for item in data.get("DataList", []):
fund = item.get("FundInfo", {}).get("Name")
bid = item.get("Price", {}).get("Bid", {}).get("Amount")
offer = item.get("Price", {}).get("Offer", {}).get("Amount")
webfunds[fund] = (
bid,
offer,
)
logging.info("Fund: {}, Bid: {}, Offer: {}".format(fund, bid, offer))
if (data.get("PageNo", 0) * PER_PAGE) < data["TotalRows"]:
page_number += 1
else:
break
return webfunds
def update_database(funddata): # pass in the list
for fundname, values in funddata.items():
bid, offer = values
if debug == 0:
funds_data.update(
{"fundname": fundname},
{
"$push": {
"prices": {
"date": str(timestamp),
"bid": bid,
"offer": offer
}
}
},
upsert=True,
)
else:
print(
"DEBUG: in update_database, would mongo $push funds_data.update webfund '",
fundname,
"' with bid: %s " % bid,
"and offer: %s " % offer,
timestamp,
)
# Ends update_database function
return
def main(scheme, ae_label, enable_sedol):
logger = get_logger()
funds_data = db[scheme]
webfunds = get_webfunds(ae_label, enable_sedol)
update_database(webfunds)
logger.info("Completed - quitting")
sys.exit(1)
\ No newline at end of file
if __name__ == "__main__":
main(scheme, ae_label, enable_sedol)
Aegon_Pension_pension.log
View file @ cd98bc4e
This diff is collapsed.
Aegon_Scraper.py
View file @ cd98bc4e
......@@ -97,15 +97,6 @@ def get_webfunds(ae_label, sedol_search):
url = uri_template.format(quote_plus(json.dumps(pl)))
try:
# Lee added for testing:
# From:
# https://blog.scrapinghub.com/python-requests-proxy
proxies = {
'http': 'http://127.0.0.1:9999',
'https': 'http://127.0.0.1:9999',
}
#page = requests.get(url, proxies=proxies)
# original continues below
page = requests.get(url)
data = json.loads(page.text)
except:
......@@ -136,12 +127,12 @@ def get_webfunds(ae_label, sedol_search):
return webfunds
def update_database(funddata): # pass in the list
for fundname, values in funddata.items():
def update_database(webfunds, funddata): # pass in the list
for fundname, values in webfunds.items():
bid, offer = values
if debug == 0:
funds_data.update(
funddata.update(
{"fundname": fundname},
{
"$push": {
......@@ -170,6 +161,6 @@ def main(scheme, ae_label, enable_sedol):
logger = get_logger()
funds_data = db[scheme]
webfunds = get_webfunds(ae_label, enable_sedol)
update_database(webfunds)
update_database(webfunds, funds_data)
logger.info("Completed - quitting")
sys.exit(1)
sys.exit(1)
\ No newline at end of file
Aegon_Scraper_config.py
View file @ cd98bc4e
debug = 1
debug = 0
scheme = "Aegon_Single_Price_Pension" # This should be overwritten when Aegon_Scraper.py is called by, Eg Aegon_Pension.py
# This should be overwritten when Aegon_Scraper.py is called by, Eg Aegon_Pension.py
scheme = None
#scheme = "Aegon_Single_Price_Pension"
#scheme = "Aegon_Pension" # This should be overwritten when Aegon_Scraper.py is called by, Eg Aegon_Pension.py
#scheme = "Aegon_Stakeholder_Pension" # This should be overwritten when Aegon_Scraper.py is called by, Eg Aegon_Pension.py
......
Aegon_Single_Price_Pension_pension.log
View file @ cd98bc4e
This diff is collapsed.
Aegon_Stakeholder_Pension_pension.log
View file @ cd98bc4e
......@@ -128,3 +128,162 @@
2020-10-13 10:34:04 Fund: With-Profits Cautious (Closed to new investors) (SH), Bid: 255.7484, Offer: 255.7484
2020-10-13 10:34:04 Fund: With-Profits Growth (Closed to new investors) (SH), Bid: 293.7605, Offer: 293.7605
2020-10-13 10:34:09 Completed - quitting
2020-10-14 06:54:43 #####################
2020-10-14 06:54:43 Starting Aegon_Stakeholder_Pension.py
2020-10-14 06:54:44 Fund: 60/40 Cautious Managed Collection (SH), Bid: 2.4219, Offer: 2.4219
2020-10-14 06:54:44 Fund: Aegon BlackRock 40/60 Global Equity Tracker (SH), Bid: 267.4106, Offer: 267.4106
2020-10-14 06:54:44 Fund: Aegon BlackRock 40/60 Global Equity Tracker Lifestyle (SH), Bid: 267.5239, Offer: 267.5239
2020-10-14 06:54:44 Fund: Aegon BlackRock 50/50 Equity and Bond Tracker (SH), Bid: 252.6252, Offer: 252.6252
2020-10-14 06:54:44 Fund: Aegon BlackRock 50/50 Global Equity Tracker (SH), Bid: 267.2997, Offer: 267.2997
2020-10-14 06:54:44 Fund: Aegon BlackRock 75/25 Equity and Bond Tracker (SH), Bid: 205.3347, Offer: 205.3347
2020-10-14 06:54:44 Fund: Aegon BlackRock 75/25 Equity and Bond Tracker Lifestyle (SH), Bid: 2.0531, Offer: 2.0531
2020-10-14 06:54:44 Fund: Aegon BlackRock Consensus (SH), Bid: 249.1317, Offer: 249.1317
2020-10-14 06:54:44 Fund: Aegon BlackRock Consensus Lifestyle (SH), Bid: 249.1393, Offer: 249.1393
2020-10-14 06:54:44 Fund: Aegon BlackRock Corporate Bond Tracker (SH), Bid: 175.0857, Offer: 175.0857
2020-10-14 06:54:44 Fund: Aegon BlackRock European Equity Tracker (SH), Bid: 224.6041, Offer: 224.6041
2020-10-14 06:54:44 Fund: Aegon BlackRock Japanese Equity Tracker (SH), Bid: 190.6444, Offer: 190.6444
2020-10-14 06:54:44 Fund: Aegon BlackRock Over 15 Years Corporate Bond Tracker (SH), Bid: 233.6638, Offer: 233.6638
2020-10-14 06:54:44 Fund: Aegon BlackRock Pacific Rim Equity Tracker (SH), Bid: 307.1229, Offer: 307.1229
2020-10-14 06:54:44 Fund: Aegon BlackRock Retirement (SH), Bid: 2.1313, Offer: 2.1313
2020-10-14 06:54:44 Fund: Aegon BlackRock UK Equity Tracker (SH), Bid: 196.6258, Offer: 196.6258
2020-10-14 06:54:44 Fund: Aegon BlackRock UK Index-Linked Gilt Tracker (SH), Bid: 265.5914, Offer: 265.5914
2020-10-14 06:54:44 Fund: Aegon BlackRock US Equity Tracker (SH), Bid: 474.1021, Offer: 474.1021
2020-10-14 06:54:44 Fund: Aegon BlackRock World (ex-UK) Equity Tracker (SH), Bid: 389.328, Offer: 389.328
2020-10-14 06:54:44 Fund: Aegon Growth Tracker (Flexible Target) (SH), Bid: 1.2025, Offer: 1.2025
2020-10-14 06:54:44 Fund: Aegon Interim Retirement (Flexible Target) (SH), Bid: 1.1113, Offer: 1.1113
2020-10-14 06:54:44 Fund: Balanced Lifestyle (SH), Bid: 268.4632, Offer: 268.4632
2020-10-14 06:54:44 Fund: Balanced Passive (SH), Bid: 246.3434, Offer: 246.3434
2020-10-14 06:54:44 Fund: Balanced Passive Lifestyle (SH), Bid: 2.488, Offer: 2.488
2020-10-14 06:54:44 Fund: Cash (SH), Bid: 131.844, Offer: 131.844
2020-10-14 06:54:44 Fund: Cautious Lifestyle (SH), Bid: 226.7798, Offer: 226.7798
2020-10-14 06:54:44 Fund: Dynamic Lifestyle (SH), Bid: 307.8243, Offer: 307.8243
2020-10-14 06:54:44 Fund: Ethical (SH), Bid: 347.6852, Offer: 347.6852
2020-10-14 06:54:44 Fund: European (SH), Bid: 218.8094, Offer: 218.8094
2020-10-14 06:54:44 Fund: European Tactical (SH), Bid: 248.1023, Offer: 248.1023
2020-10-14 06:54:44 Fund: Global (SH), Bid: 307.8218, Offer: 307.8218
2020-10-14 06:54:44 Fund: Global Equity Tracker (SH), Bid: 334.2961, Offer: 334.2961
2020-10-14 06:54:44 Fund: Global Equity Tracker Lifestyle (SH), Bid: 334.3376, Offer: 334.3376
2020-10-14 06:54:44 Fund: High Yield Corporate Bond (SH), Bid: 266.6263, Offer: 266.6263
2020-10-14 06:54:44 Fund: Index-Linked (SH), Bid: 326.7959, Offer: 326.7959
2020-10-14 06:54:44 Fund: International (SH), Bid: 312.5464, Offer: 312.5464
2020-10-14 06:54:44 Fund: Japan (SH), Bid: 149.75, Offer: 149.75
2020-10-14 06:54:44 Fund: Long Gilt (SH), Bid: 319.6069, Offer: 319.6069
2020-10-14 06:54:44 Fund: Mixed (SH), Bid: 268.4817, Offer: 268.4817
2020-10-14 06:54:44 Fund: North American (SH), Bid: 256.1477, Offer: 256.1477
2020-10-14 06:54:44 Fund: Overseas Bond (SH), Bid: 227.2398, Offer: 227.2398
2020-10-14 06:54:44 Fund: Overseas Equity Tracker (SH), Bid: 409.7843, Offer: 409.7843
2020-10-14 06:54:44 Fund: Pacific (SH), Bid: 2236.915, Offer: 2236.915
2020-10-14 06:54:44 Fund: Scottish Equitable Lazard European Smaller Companies (SH), Bid: 5.2093, Offer: 5.2093
2020-10-14 06:54:44 Fund: Scottish Equitable Lifestyle (SH), Bid: 4.9326, Offer: 4.9326
2020-10-14 06:54:44 Fund: Scottish Equitable Retirement (SH), Bid: 263.3605, Offer: 263.3605
2020-10-14 06:54:44 Fund: Stakeholder Default (SH), Bid: 2.3996, Offer: 2.3996
2020-10-14 06:54:44 Fund: Technology (SH), Bid: 539.2028, Offer: 539.2028
2020-10-14 06:54:44 Fund: UK Corporate Bond (SH), Bid: 257.7146, Offer: 257.7146
2020-10-14 06:54:44 Fund: UK Equity (SH), Bid: 218.979, Offer: 218.979
2020-10-14 06:54:44 Fund: UK Equity Tactical (SH), Bid: 231.6634, Offer: 231.6634
2020-10-14 06:54:44 Fund: UK Fixed Interest (SH), Bid: 254.4655, Offer: 254.4655
2020-10-14 06:54:44 Fund: UK Fixed Interest and Global Equity Tracker (SH), Bid: 277.9058, Offer: 277.9058
2020-10-14 06:54:44 Fund: UK Fixed Interest and Global Equity Tracker Lifestyle (SH), Bid: 277.8992, Offer: 277.8992
2020-10-14 06:54:44 Fund: UK Government Bond (SH), Bid: 239.1568, Offer: 239.1568
2020-10-14 06:54:44 Fund: UK Index Tracker (SH), Bid: 201.5412, Offer: 201.5412
2020-10-14 06:54:44 Fund: UK Long Corporate Bond (SH), Bid: 330.8717, Offer: 330.8717
2020-10-14 06:54:44 Fund: UK Smaller Companies (SH), Bid: 539.2219, Offer: 539.2219
2020-10-14 06:54:44 Fund: Universal Balanced Collection (SH), Bid: 267.6845, Offer: 267.6845
2020-10-14 06:54:44 Fund: Universal Lifestyle Collection (SH), Bid: 254.2912, Offer: 254.2912
2020-10-14 06:54:44 Fund: With-Profits Cautious (Closed to new investors) (SH), Bid: 255.7471, Offer: 255.7471
2020-10-14 06:54:44 Fund: With-Profits Growth (Closed to new investors) (SH), Bid: 293.5105, Offer: 293.5105
2020-10-14 06:54:45 #####################
2020-10-14 06:54:45 Starting Aegon_Stakeholder_Pension.py
2020-10-14 06:54:45 Traceback (most recent call last):
File "/home/lee/bs_price_grabbers/2020_Aegon/Aegon_Scraper.py", line 100, in get_webfunds
page = requests.get(url)
File "/home/lee/py3_env/lib/python3.6/site-packages/requests/api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "/home/lee/py3_env/lib/python3.6/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/lee/py3_env/lib/python3.6/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/home/lee/py3_env/lib/python3.6/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/home/lee/py3_env/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/lee/py3_env/lib/python3.6/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/home/lee/py3_env/lib/python3.6/site-packages/urllib3/connectionpool.py", line 381, in _make_request
self._validate_conn(conn)
File "/home/lee/py3_env/lib/python3.6/site-packages/urllib3/connectionpool.py", line 978, in _validate_conn
conn.connect()
File "/home/lee/py3_env/lib/python3.6/site-packages/urllib3/connection.py", line 309, in connect
conn = self._new_conn()
File "/home/lee/py3_env/lib/python3.6/site-packages/urllib3/connection.py", line 160, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw
File "/home/lee/py3_env/lib/python3.6/site-packages/urllib3/util/connection.py", line 74, in create_connection
sock.connect(sa)
KeyboardInterrupt
2020-10-14 06:54:45 Error downloading page at: https://digitalfundservice.feprecisionplus.com/FundDataService.svc/GetUnitList?jsonString=%7B%22FilteringOptions%22%3A+%7B%22undefined%22%3A+0%2C+%22SearchText%22%3A+%22%22%2C+%22Ter%22%3A+%7B%7D%2C+%22RangeId%22%3A+null%2C+%22RangeName%22%3A+%22%22%2C+%22CategoryId%22%3A+null%2C+%22PriipProductCode%22%3A+null%2C+%22DefaultCategoryId%22%3A+null%2C+%22ForSaleIn%22%3A+null%2C+%22ShowMainUnits%22%3A+false%2C+%22MPCategoryCode%22%3A+null%7D%2C+%22ProjectName%22%3A+%22aegonpensionssh%22%2C+%22LanguageCode%22%3A+%22en-GB%22%2C+%22LanguageId%22%3A+%221%22%2C+%22Theme%22%3A+%22aegonp%22%2C+%22SortingStyle%22%3A+%221%22%2C+%22PageNo%22%3A+4%2C+%22PageSize%22%3A+50%2C+%22OrderBy%22%3A+%22UnitName%3Ainit%22%2C+%22IsAscOrder%22%3A+true%2C+%22OverrideDocumentCountryCode%22%3A+null%2C+%22ToolId%22%3A+%221%22%2C+%22PrefetchPages%22%3A+40%2C+%22PrefetchPageStart%22%3A+1%2C+%22OverridenThemeName%22%3A+%22aegonp%22%2C+%22ForSaleIn%22%3A+%22%22%2C+%22ValidateFeResearchAccess%22%3A+false%2C+%22HasFeResearchFullAccess%22%3A+false%2C+%22EnableSedolSearch%22%3A+false%2C+%22RowCount%22%3A+0%2C+%22RowIDs%22%3A+%22151%2C152%2C153%2C154%2C155%2C156%2C157%2C158%2C159%2C160%2C161%2C162%2C163%2C164%2C165%2C166%2C167%2C168%2C169%2C170%2C171%2C172%2C173%2C174%2C175%2C176%2C177%2C178%2C179%2C180%2C181%2C182%2C183%2C184%2C185%2C186%2C187%2C188%2C189%2C190%2C191%2C192%2C193%2C194%2C195%2C196%2C197%2C198%2C199%2C200%22%7D
2020-10-14 06:54:48 #####################
2020-10-14 06:54:48 Starting Aegon_Stakeholder_Pension.py
2020-10-14 06:54:48 Fund: 60/40 Cautious Managed Collection (SH), Bid: 2.4219, Offer: 2.4219
2020-10-14 06:54:48 Fund: Aegon BlackRock 40/60 Global Equity Tracker (SH), Bid: 267.4106, Offer: 267.4106
2020-10-14 06:54:48 Fund: Aegon BlackRock 40/60 Global Equity Tracker Lifestyle (SH), Bid: 267.5239, Offer: 267.5239
2020-10-14 06:54:48 Fund: Aegon BlackRock 50/50 Equity and Bond Tracker (SH), Bid: 252.6252, Offer: 252.6252
2020-10-14 06:54:48 Fund: Aegon BlackRock 50/50 Global Equity Tracker (SH), Bid: 267.2997, Offer: 267.2997
2020-10-14 06:54:48 Fund: Aegon BlackRock 75/25 Equity and Bond Tracker (SH), Bid: 205.3347, Offer: 205.3347
2020-10-14 06:54:48 Fund: Aegon BlackRock 75/25 Equity and Bond Tracker Lifestyle (SH), Bid: 2.0531, Offer: 2.0531
2020-10-14 06:54:48 Fund: Aegon BlackRock Consensus (SH), Bid: 249.1317, Offer: 249.1317
2020-10-14 06:54:48 Fund: Aegon BlackRock Consensus Lifestyle (SH), Bid: 249.1393, Offer: 249.1393
2020-10-14 06:54:48 Fund: Aegon BlackRock Corporate Bond Tracker (SH), Bid: 175.0857, Offer: 175.0857
2020-10-14 06:54:48 Fund: Aegon BlackRock European Equity Tracker (SH), Bid: 224.6041, Offer: 224.6041
2020-10-14 06:54:48 Fund: Aegon BlackRock Japanese Equity Tracker (SH), Bid: 190.6444, Offer: 190.6444
2020-10-14 06:54:48 Fund: Aegon BlackRock Over 15 Years Corporate Bond Tracker (SH), Bid: 233.6638, Offer: 233.6638
2020-10-14 06:54:48 Fund: Aegon BlackRock Pacific Rim Equity Tracker (SH), Bid: 307.1229, Offer: 307.1229
2020-10-14 06:54:48 Fund: Aegon BlackRock Retirement (SH), Bid: 2.1313, Offer: 2.1313
2020-10-14 06:54:48 Fund: Aegon BlackRock UK Equity Tracker (SH), Bid: 196.6258, Offer: 196.6258
2020-10-14 06:54:48 Fund: Aegon BlackRock UK Index-Linked Gilt Tracker (SH), Bid: 265.5914, Offer: 265.5914
2020-10-14 06:54:48 Fund: Aegon BlackRock US Equity Tracker (SH), Bid: 474.1021, Offer: 474.1021
2020-10-14 06:54:48 Fund: Aegon BlackRock World (ex-UK) Equity Tracker (SH), Bid: 389.328, Offer: 389.328
2020-10-14 06:54:48 Fund: Aegon Growth Tracker (Flexible Target) (SH), Bid: 1.2025, Offer: 1.2025
2020-10-14 06:54:48 Fund: Aegon Interim Retirement (Flexible Target) (SH), Bid: 1.1113, Offer: 1.1113
2020-10-14 06:54:48 Fund: Balanced Lifestyle (SH), Bid: 268.4632, Offer: 268.4632
2020-10-14 06:54:48 Fund: Balanced Passive (SH), Bid: 246.3434, Offer: 246.3434
2020-10-14 06:54:48 Fund: Balanced Passive Lifestyle (SH), Bid: 2.488, Offer: 2.488
2020-10-14 06:54:48 Fund: Cash (SH), Bid: 131.844, Offer: 131.844
2020-10-14 06:54:48 Fund: Cautious Lifestyle (SH), Bid: 226.7798, Offer: 226.7798
2020-10-14 06:54:48 Fund: Dynamic Lifestyle (SH), Bid: 307.8243, Offer: 307.8243
2020-10-14 06:54:48 Fund: Ethical (SH), Bid: 347.6852, Offer: 347.6852
2020-10-14 06:54:48 Fund: European (SH), Bid: 218.8094, Offer: 218.8094
2020-10-14 06:54:48 Fund: European Tactical (SH), Bid: 248.1023, Offer: 248.1023
2020-10-14 06:54:48 Fund: Global (SH), Bid: 307.8218, Offer: 307.8218
2020-10-14 06:54:48 Fund: Global Equity Tracker (SH), Bid: 334.2961, Offer: 334.2961
2020-10-14 06:54:48 Fund: Global Equity Tracker Lifestyle (SH), Bid: 334.3376, Offer: 334.3376
2020-10-14 06:54:48 Fund: High Yield Corporate Bond (SH), Bid: 266.6263, Offer: 266.6263
2020-10-14 06:54:48 Fund: Index-Linked (SH), Bid: 326.7959, Offer: 326.7959
2020-10-14 06:54:48 Fund: International (SH), Bid: 312.5464, Offer: 312.5464
2020-10-14 06:54:48 Fund: Japan (SH), Bid: 149.75, Offer: 149.75
2020-10-14 06:54:48 Fund: Long Gilt (SH), Bid: 319.6069, Offer: 319.6069
2020-10-14 06:54:48 Fund: Mixed (SH), Bid: 268.4817, Offer: 268.4817
2020-10-14 06:54:48 Fund: North American (SH), Bid: 256.1477, Offer: 256.1477
2020-10-14 06:54:48 Fund: Overseas Bond (SH), Bid: 227.2398, Offer: 227.2398
2020-10-14 06:54:48 Fund: Overseas Equity Tracker (SH), Bid: 409.7843, Offer: 409.7843
2020-10-14 06:54:48 Fund: Pacific (SH), Bid: 2236.915, Offer: 2236.915
2020-10-14 06:54:48 Fund: Scottish Equitable Lazard European Smaller Companies (SH), Bid: 5.2093, Offer: 5.2093
2020-10-14 06:54:48 Fund: Scottish Equitable Lifestyle (SH), Bid: 4.9326, Offer: 4.9326
2020-10-14 06:54:48 Fund: Scottish Equitable Retirement (SH), Bid: 263.3605, Offer: 263.3605
2020-10-14 06:54:48 Fund: Stakeholder Default (SH), Bid: 2.3996, Offer: 2.3996
2020-10-14 06:54:48 Fund: Technology (SH), Bid: 539.2028, Offer: 539.2028
2020-10-14 06:54:48 Fund: UK Corporate Bond (SH), Bid: 257.7146, Offer: 257.7146
2020-10-14 06:54:48 Fund: UK Equity (SH), Bid: 218.979, Offer: 218.979
2020-10-14 06:54:48 Fund: UK Equity Tactical (SH), Bid: 231.6634, Offer: 231.6634
2020-10-14 06:54:48 Fund: UK Fixed Interest (SH), Bid: 254.4655, Offer: 254.4655
2020-10-14 06:54:48 Fund: UK Fixed Interest and Global Equity Tracker (SH), Bid: 277.9058, Offer: 277.9058
2020-10-14 06:54:48 Fund: UK Fixed Interest and Global Equity Tracker Lifestyle (SH), Bid: 277.8992, Offer: 277.8992
2020-10-14 06:54:48 Fund: UK Government Bond (SH), Bid: 239.1568, Offer: 239.1568
2020-10-14 06:54:48 Fund: UK Index Tracker (SH), Bid: 201.5412, Offer: 201.5412
2020-10-14 06:54:48 Fund: UK Long Corporate Bond (SH), Bid: 330.8717, Offer: 330.8717
2020-10-14 06:54:48 Fund: UK Smaller Companies (SH), Bid: 539.2219, Offer: 539.2219
2020-10-14 06:54:48 Fund: Universal Balanced Collection (SH), Bid: 267.6845, Offer: 267.6845
2020-10-14 06:54:48 Fund: Universal Lifestyle Collection (SH), Bid: 254.2912, Offer: 254.2912
2020-10-14 06:54:48 Fund: With-Profits Cautious (Closed to new investors) (SH), Bid: 255.7471, Offer: 255.7471
2020-10-14 06:54:48 Fund: With-Profits Growth (Closed to new investors) (SH), Bid: 293.5105, Offer: 293.5105
2020-10-14 06:54:53 Completed - quitting
Igor_supplied_Aegon_Pension.py deleted 100755 → 0
View file @ 7e3afd83
#!/usr/bin/env python
from Aegon_Scraper import main
ae_label = "aegonportal"
enable_sedol = True
scheme = "Aegon_Pension"
if __name__ == "__main__":
main(scheme, ae_label, enable_sedol)
test_archive/Lee modified Aegon_Pension.py 0 → 100755
View file @ cd98bc4e
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
from urllib.parse import quote_plus, urlencode
from datetime import date
from pymongo import MongoClient
import json
import importlib
import requests
import traceback
my_module = importlib.import_module(
os.path.splitext((os.path.basename(__file__)))[0] + "_config"
)
debug = my_module.debug
uri_template = my_module.uri_template
payload = my_module.payload
headers = my_module.headers
scheme = my_module.scheme
database = my_module.database
mongoServer = my_module.mongoServer
mongoPort = my_module.mongoPort
PER_PAGE = my_module.PER_PAGE
MAX_ROWS = my_module.MAX_ROWS
connection = MongoClient(mongoServer, mongoPort)
db = connection[database]
def get_logger():
# Set up logging
file_path = os.path.basename(sys.argv[0])
filename = file_path.replace(".py", "")
# logfile = "/tmp/" + filename + "_pension.log"
logfile = filename + "_pension.log"
logging.basicConfig(
filename=logfile,
format="%(asctime)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO,
)
logger = logging.getLogger(__name__)
logger.info("#####################")
logger.info("Starting %s", file_path)
return logger
def get_timestamp():
today = date.today().toordinal()
lastWeek = today - 7
sunday = lastWeek - (lastWeek % 7)
friday = sunday + 5
timestamp = date.fromordinal(friday)
return timestamp
timestamp = get_timestamp()
def get_dbfunds(funds_data):
dbfunds = {}
for doc in funds_data.find({}):
fundname = doc["fundname"]
last_data = doc["prices"][-1]
price_string = (last_data["sell_price"], last_data["buy_price"])
dbfunds[fundname] = price_string
return dbfunds
def get_webfunds(ae_label, sedol_search):
webfunds = dict()
total = 0
page_number = 1
while True:
pl = payload.copy()
pl["PageNo"] = page_number
pl["PageSize"] = PER_PAGE
pl["RowIDs"] = ",".join(
[
str(i)
for i in range(
PER_PAGE * (page_number - 1) + 1, page_number * PER_PAGE + 1
)
]
)
pl["ProjectName"] = ae_label
pl["EnableSedolSearch"] = sedol_search
url = uri_template.format(quote_plus(json.dumps(pl)))
try:
page = requests.get(url)
data = json.loads(page.text)
except:
logger = get_logger()
logger.error(traceback.format_exc())
logging.error("Error downloading page at: " + url)
sys.exit(1)
else:
data = json.loads(data)
data["TotalRows"] = data.get("TotalRows") or MAX_ROWS
for item in data.get("DataList", []):
fund = item.get("FundInfo", {}).get("Name")
bid = item.get("Price", {}).get("Bid", {}).get("Amount")
offer = item.get("Price", {}).get("Offer", {}).get("Amount")
webfunds[fund] = (
bid,
offer,
)
logging.info("Fund: {}, Bid: {}, Offer: {}".format(fund, bid, offer))
if (data.get("PageNo", 0) * PER_PAGE) < data["TotalRows"]:
page_number += 1
else:
break
return webfunds
def update_database(funddata): # pass in the list
for fundname, values in funddata.items():
bid, offer = values
if debug == 0:
funds_data.update(
{"fundname": fundname},
{
"$push": {
"prices": {
"date": str(timestamp),
"bid": bid,
"offer": offer
}
}
},
upsert=True,
)
else:
print(
"DEBUG: in update_database, would mongo $push funds_data.update webfund '",
fundname,
"' with bid: %s " % bid,
"and offer: %s " % offer,
timestamp,
)
# Ends update_database function
return
def main(scheme, ae_label, enable_sedol):
logger = get_logger()
funds_data = db[scheme]
webfunds = get_webfunds(ae_label, enable_sedol)
update_database(webfunds)
logger.info("Completed - quitting")
sys.exit(1)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment