AccountStats Plugin - daily notification (#349)

* Adding tools dir and a simple script to analyse lending history
* Implement AccountStats plugin - WORK IN PROGRESS
- Created Plugin system, allowing user to define which Plugins are loaded from config file
- Started implementation of AccountStats (fetching history)

* Fetch all lending history from Poloniex
Split Plugin events to before and after lending
* Fix fetch all lending history from Poloniex
Added notify event (every cycle at the moment)
* Notify every 24Hrs
* Updated documentation
* Removed tools
* Strip plugin names from config

Author: rnevet

docs/configuration.rst

@@ -295,6 +295,26 @@ Advanced logging and Web Display
     - Acceptable values: BTC, USDT, Any coin with a direct Poloniex BTC trading pair (ex. DOGE, MAID, ETH), Currencies that have a BTC exchange rate on blockchain.info (i.e. EUR, USD)
     - Will be a close estimate, due to unexpected market fluctuations, trade fees, and other unforseeable factors.
 
+Plugins
+-------
+
+Plugins allow extending Bot functionality with extra features.
+To enable/disable a plugin add/remove it to the ``plugins`` list config option, example::
+
+    plugins = Plugin1, Plugin2, etc...
+
+AccountStats Plugin
+~~~~~~~~~~~~~~~~~~~
+
+The AccountStats plugin fetches all your loan history and provides statistics based on it.
+Current implementation sends a earnings summary Notification (see Notifications sections) every 24hr.
+
+To enable the plugin add ``AccountStats`` to the ``plugins`` config options, example::
+
+    plugins = AccountStats
+
+Be aware that first initialization might take longer as the bot will fetch all the history.
+
 lendingbot.html options
 -----------------------
 

lendingbot.py

@@ -4,17 +4,18 @@
 import sys
 import time
 import traceback
+from decimal import Decimal
 from httplib import BadStatusLine
 from urllib2 import URLError
 
-from decimal import Decimal
-
-from modules.Logger import Logger
-from modules.Poloniex import Poloniex, PoloniexApiError
 import modules.Configuration as Config
-import modules.MaxToLend as MaxToLend
 import modules.Data as Data
 import modules.Lending as Lending
+import modules.MaxToLend as MaxToLend
+from modules.Logger import Logger
+from modules.Poloniex import Poloniex, PoloniexApiError
+import modules.PluginsManager as PluginsManager
+
 
 try:
     open('lendingbot.py', 'r')
@@ -57,6 +58,8 @@
     analysis = None
 Lending.init(Config, api, log, Data, MaxToLend, dry_run, analysis, notify_conf)
 
+# load plugins
+PluginsManager.init(Config, api, log, notify_conf)
 
 print 'Welcome to Poloniex Lending Bot'
 # Configure web server
@@ -70,9 +73,11 @@
     while True:
         try:
             Data.update_conversion_rates(output_currency, json_output_enabled)
+            PluginsManager.before_lending()
             Lending.transfer_balances()
             Lending.cancel_all()
             Lending.lend_all()
+            PluginsManager.after_lending()
             log.refreshStatus(Data.stringify_total_lent(*Data.get_total_lent()),
                               Data.get_max_duration(end_date, "status"))
             log.persistStatus()
@@ -116,6 +121,7 @@
 except KeyboardInterrupt:
     if web_server_enabled:
         WebServer.stop_web_server()
+    PluginsManager.on_bot_exit()
     log.log('bye')
     print 'bye'
     os._exit(0)  # Ad-hoc solution in place of 'exit(0)' TODO: Find out why non-daemon thread(s) are hanging on exit

modules/Configuration.py

@@ -158,3 +158,10 @@ def get_notification_config():
             notify_conf[conf] = get('notifications', conf)
 
     return notify_conf
+
+
+def get_plugins_config():
+    active_plugins = []
+    if config.has_option("BOT", "plugins"):
+        active_plugins = map(str.strip, config.get("BOT", "plugins").split(','))
+    return active_plugins

modules/PluginsManager.py

@@ -0,0 +1,51 @@
+from plugins import *
+import plugins.Plugin as Plugin
+
+config = None
+api = None
+log = None
+notify_conf = None
+plugins = []
+
+
+def init_plugin(plugin_name):
+    """
+        :return: instance of requested class
+        :rtype: Plugin
+        """
+    klass = globals()[plugin_name]  # type: Plugin
+    instance = klass(config, api, log, notify_conf)
+    instance.on_bot_init()
+    return instance
+
+
+def init(cfg, api1, log1, notify_conf1):
+    """
+    @type cfg1: modules.Configuration
+    @type api1: modules.Poloniex.Poloniex
+    @type log1: modules.Logger.Logger
+    """
+    global config, api, log, notify_conf
+    config = cfg
+    api = api1
+    log = log1
+    notify_conf = notify_conf1
+
+    plugin_names = config.get_plugins_config()
+    for plugin_name in plugin_names:
+        plugins.append(init_plugin(plugin_name))
+
+
+def after_lending():
+    for plugin in plugins:
+        plugin.after_lending()
+
+
+def before_lending():
+    for plugin in plugins:
+        plugin.before_lending()
+
+
+def on_bot_exit():
+    for plugin in plugins:
+        plugin.on_bot_exit()

modules/Poloniex.py

@@ -7,6 +7,8 @@
 import urllib
 import urllib2
 import threading
+import calendar
+
 from modules.RingBuffer import RingBuffer
 
 
@@ -15,7 +17,7 @@ class PoloniexApiError(Exception):
 
 
 def create_time_stamp(datestr, formatting="%Y-%m-%d %H:%M:%S"):
-    return time.mktime(time.strptime(datestr, formatting))
+    return calendar.timegm(time.strptime(datestr, formatting))
 
 
 def post_process(before):
@@ -175,6 +177,9 @@ def return_open_loan_offers(self):
     def return_active_loans(self):
         return self.api_query('returnActiveLoans')
 
+    def return_lending_history(self, start, stop, limit=500):
+        return self.api_query('returnLendingHistory', {'start': start, 'end': stop, 'limit': limit})
+
     # Returns your trade history for a given market, specified by the "currencyPair" POST parameter
     # Inputs:
     # currencyPair  The currency pair e.g. "BTC_XCP"

plugins/AccountStats.py

@@ -0,0 +1,122 @@
+# coding=utf-8
+from plugins.Plugin import Plugin
+import modules.Poloniex as Poloniex
+import sqlite3
+
+BITCOIN_GENESIS_BLOCK_DATE = "2009-01-03 18:15:05"
+DAY_IN_SEC = 86400
+DB_DROP = "DROP TABLE IF EXISTS history"
+DB_CREATE = "CREATE TABLE IF NOT EXISTS history(" \
+            "id INTEGER UNIQUE, open TIMESTAMP, close TIMESTAMP," \
+            " duration NUMBER, interest NUMBER, rate NUMBER," \
+            " currency TEXT, amount NUMBER, earned NUMBER, fee NUMBER )"
+DB_INSERT = "INSERT OR REPLACE INTO 'history'" \
+            "('id','open','close','duration','interest','rate','currency','amount','earned','fee')" \
+            " VALUES (?,?,?,?,?,?,?,?,?,?);"
+DB_GET_LAST_TIMESTAMP = "SELECT max(close) as last_timestamp FROM 'history'"
+DB_GET_FIRST_TIMESTAMP = "SELECT min(close) as first_timestamp FROM 'history'"
+DB_GET_TOTAL_EARNED = "SELECT sum(earned) as total_earned, currency FROM 'history' GROUP BY currency"
+DB_GET_24HR_EARNED = "SELECT sum(earned) as total_earned, currency FROM 'history' " \
+                     "WHERE close BETWEEN datetime('now','-1 day') AND datetime('now') GROUP BY currency"
+
+
+class AccountStats(Plugin):
+    last_notification = 0
+
+    def on_bot_init(self):
+        super(AccountStats, self).on_bot_init()
+        self.init_db()
+
+    def after_lending(self):
+        self.update_history()
+        self.notify_daily()
+
+    # noinspection PyAttributeOutsideInit
+    def init_db(self):
+        self.db = sqlite3.connect(r'market_data\loan_history.sqlite3')
+        self.db.execute(DB_CREATE)
+        self.db.commit()
+
+    def update_history(self):
+        # timestamps are in UTC
+        last_time_stamp = self.get_last_timestamp()
+
+        if last_time_stamp is None:
+            # no entries means db is empty and needs initialization
+            last_time_stamp = BITCOIN_GENESIS_BLOCK_DATE
+            self.db.execute("PRAGMA user_version = 0")
+
+        self.fetch_history(Poloniex.create_time_stamp(last_time_stamp), sqlite3.time.time())
+
+        # As Poloniex API return a unspecified number of recent loans, but not all so we need to loop back.
+        if (self.get_db_version() == 0) and (self.get_first_timestamp() is not None):
+            last_time_stamp = BITCOIN_GENESIS_BLOCK_DATE
+            loop = True
+            while loop:
+                sqlite3.time.sleep(10)  # delay a bit, try not to annoy poloniex
+                first_time_stamp = self.get_first_timestamp()
+                count = self.fetch_history(Poloniex.create_time_stamp(last_time_stamp, )
+                                           , Poloniex.create_time_stamp(first_time_stamp))
+                loop = count != 0
+            # if we reached here without errors means we managed to fetch all the history, db is ready.
+            self.set_db_version(1)
+
+    def set_db_version(self, version):
+        self.db.execute("PRAGMA user_version = " + str(version))
+
+    def get_db_version(self):
+        return self.db.execute("PRAGMA user_version").fetchone()[0]
+
+    def fetch_history(self, first_time_stamp, last_time_stamp):
+        history = self.api.return_lending_history(first_time_stamp, last_time_stamp - 1, 50000)
+        loans = []
+        for loan in reversed(history):
+            loans.append(
+                [loan['id'], loan['open'], loan['close'], loan['duration'], loan['interest'],
+                 loan['rate'], loan['currency'], loan['amount'], loan['earned'], loan['fee']])
+        self.db.executemany(DB_INSERT, loans)
+        self.db.commit()
+        count = len(loans)
+        self.log.log('Downloaded ' + str(count) + ' loans history '
+                     + sqlite3.datetime.datetime.utcfromtimestamp(first_time_stamp).strftime('%Y-%m-%d %H:%M:%S')
+                     + ' to ' + sqlite3.datetime.datetime.utcfromtimestamp(last_time_stamp - 1).strftime(
+            '%Y-%m-%d %H:%M:%S'))
+        if count > 0:
+            self.log.log('Last: ' + history[0]['close'] + ' First:' + history[count - 1]['close'])
+        return count
+
+    def get_last_timestamp(self):
+        cursor = self.db.execute(DB_GET_LAST_TIMESTAMP)
+        row = cursor.fetchone()
+        cursor.close()
+        return row[0]
+
+    def get_first_timestamp(self):
+        cursor = self.db.execute(DB_GET_FIRST_TIMESTAMP)
+        row = cursor.fetchone()
+        cursor.close()
+        return row[0]
+
+    def notify_daily(self):
+        if self.get_db_version() == 0:
+            self.log.log_error('AccountStats DB isn\'t ready.')
+            return
+
+        if self.last_notification != 0 and self.last_notification + DAY_IN_SEC > sqlite3.time.time():
+            return
+
+        cursor = self.db.execute(DB_GET_24HR_EARNED)
+        output = ''
+        for row in cursor:
+            output += str(row[0]) + ' ' + str(row[1]) + ' in last 24hrs\n'
+        cursor.close()
+
+        cursor = self.db.execute(DB_GET_TOTAL_EARNED)
+        for row in cursor:
+            output += str(row[0]) + ' ' + str(row[1]) + ' in total\n'
+        cursor.close()
+        if output != '':
+            self.last_notification = sqlite3.time.time()
+            output = 'Earnings:\n----------\n' + output
+            self.log.notify(output, self.notify_config)
+            self.log.log(output)

plugins/Plugin.py

@@ -0,0 +1,31 @@
+# coding=utf-8
+
+
+class Plugin(object):
+    """
+    @type cfg1: modules.Configuration
+    @type api1: modules.Poloniex.Poloniex
+    @type log1: modules.Logger.Logger
+    """
+    def __init__(self, cfg1, api1, log1, notify_config1):
+        self.api = api1
+        self.config = cfg1
+        self.notify_config = notify_config1
+        self.log = log1
+
+    # override this to run plugin init code
+    def on_bot_init(self):
+        self.log.log(self.__class__.__name__ + ' plugin started.')
+
+    # override this to run plugin loop code before lending
+    def before_lending(self):
+        pass
+
+    # override this to run plugin loop code after lending
+    def after_lending(self):
+        pass
+
+    # override this to run plugin stop code
+    # since the bot can be killed, there is not guarantee this will be called.
+    def on_bot_stop(self):
+        pass

plugins/__init__.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+__all__ = []
+
+import pkgutil
+import inspect
+
+for loader, name, is_pkg in pkgutil.walk_packages(__path__):
+    module = loader.find_module(name).load_module(name)
+
+    for name, value in inspect.getmembers(module):
+        if name.startswith('__'):
+            continue
+
+        globals()[name] = value
+        __all__.append(name)