├── xls ├── cron1.xls ├── cron2.xls ├── cron3.xls ├── wsock1.xls ├── quandl1.xls ├── quandl2.xls ├── quandl3.xls ├── tiingo1.xls ├── tiingo2.xls ├── tiingows1.xls ├── quandl1_proxy.xls ├── quandl2_proxy.xls ├── quandl3_proxy.xls ├── tiingo2_proxy.xls ├── tiingows1_proxy.xls ├── tiingows_option1.xls ├── baremetrics_plan1.xlsx ├── flight_bare_model1.xls ├── flight_bare_model2.xls ├── baremetrics_metric1.xlsx ├── baremetrics_summary1.xlsx ├── flight_bare_model1.xlsx ├── ganalytics_sessions1.xlsx ├── ganalytics_sessions2.xls ├── ganalytics_sessions2.xlsx ├── ganalytics_sessions3.xls └── tiingows_option1_proxy.xls ├── src ├── bin │ ├── Debug │ │ ├── SSAddin.xll │ │ ├── SSAddin64.xll │ │ └── SSAddin.dna │ └── Release │ │ ├── SSAddin.xll │ │ └── SSAddin64.xll ├── Properties │ └── AssemblyInfo.cs ├── Packages │ ├── Excel-DNA.0.30.3 │ │ └── content │ │ │ └── ExcelDna-Template.dna │ ├── Excel-DNA.0.32.0 │ │ └── content │ │ │ └── ExcelDna-Template.dna │ └── Packages.config ├── SSAddin.xll.config ├── ssaddin.dna ├── AddIn.cs ├── Logr.cs ├── TiingoRealTimeMessageHandler.cs ├── ProxyConnectorBase.cs ├── WSCallback.cs ├── GoogleAnalyticsAPI.cs ├── RTDServer.cs ├── JsonToDictionary.cs ├── ssaddin.csproj ├── HttpConnectProxy.cs ├── DataCache.cs ├── CronManager.cs ├── TWSCallback.cs ├── WorksheetFunctions.cs ├── ConfigSheet.cs └── SSWebClient.cs ├── docs ├── index.rst ├── install.rst ├── config.rst ├── make.bat ├── functions.rst └── conf.py ├── cfg └── SSAddin.xll.config ├── README.md └── LICENSE /xls/cron1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/cron1.xls -------------------------------------------------------------------------------- /xls/cron2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/cron2.xls -------------------------------------------------------------------------------- /xls/cron3.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/cron3.xls -------------------------------------------------------------------------------- /xls/wsock1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/wsock1.xls -------------------------------------------------------------------------------- /xls/quandl1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/quandl1.xls -------------------------------------------------------------------------------- /xls/quandl2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/quandl2.xls -------------------------------------------------------------------------------- /xls/quandl3.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/quandl3.xls -------------------------------------------------------------------------------- /xls/tiingo1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/tiingo1.xls -------------------------------------------------------------------------------- /xls/tiingo2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/tiingo2.xls -------------------------------------------------------------------------------- /xls/tiingows1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/tiingows1.xls -------------------------------------------------------------------------------- /xls/quandl1_proxy.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/quandl1_proxy.xls -------------------------------------------------------------------------------- /xls/quandl2_proxy.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/quandl2_proxy.xls -------------------------------------------------------------------------------- /xls/quandl3_proxy.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/quandl3_proxy.xls -------------------------------------------------------------------------------- /xls/tiingo2_proxy.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/tiingo2_proxy.xls -------------------------------------------------------------------------------- /xls/tiingows1_proxy.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/tiingows1_proxy.xls -------------------------------------------------------------------------------- /xls/tiingows_option1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/tiingows_option1.xls -------------------------------------------------------------------------------- /src/bin/Debug/SSAddin.xll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/src/bin/Debug/SSAddin.xll -------------------------------------------------------------------------------- /xls/baremetrics_plan1.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/baremetrics_plan1.xlsx -------------------------------------------------------------------------------- /xls/flight_bare_model1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/flight_bare_model1.xls -------------------------------------------------------------------------------- /xls/flight_bare_model2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/flight_bare_model2.xls -------------------------------------------------------------------------------- /src/bin/Debug/SSAddin64.xll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/src/bin/Debug/SSAddin64.xll -------------------------------------------------------------------------------- /src/bin/Release/SSAddin.xll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/src/bin/Release/SSAddin.xll -------------------------------------------------------------------------------- /src/bin/Release/SSAddin64.xll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/src/bin/Release/SSAddin64.xll -------------------------------------------------------------------------------- /xls/baremetrics_metric1.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/baremetrics_metric1.xlsx -------------------------------------------------------------------------------- /xls/baremetrics_summary1.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/baremetrics_summary1.xlsx -------------------------------------------------------------------------------- /xls/flight_bare_model1.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/flight_bare_model1.xlsx -------------------------------------------------------------------------------- /xls/ganalytics_sessions1.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/ganalytics_sessions1.xlsx -------------------------------------------------------------------------------- /xls/ganalytics_sessions2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/ganalytics_sessions2.xls -------------------------------------------------------------------------------- /xls/ganalytics_sessions2.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/ganalytics_sessions2.xlsx -------------------------------------------------------------------------------- /xls/ganalytics_sessions3.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/ganalytics_sessions3.xls -------------------------------------------------------------------------------- /xls/tiingows_option1_proxy.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpreadServe/SSAddin/HEAD/xls/tiingows_option1_proxy.xls -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. SpreadServe Addin documentation master file, created by 2 | sphinx-quickstart on Thu Aug 06 13:34:13 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | SpreadServe Addin 7 | ================= 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | install 15 | functions 16 | config 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /src/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | [assembly: AssemblyTitle("SSAddin")] 5 | [assembly: AssemblyDescription("SSAddin Excel Add-In")] 6 | [assembly: AssemblyCompany("Babbington Slade Ltd")] 7 | [assembly: AssemblyProduct("SSAddin Excel Add-In")] 8 | [assembly: AssemblyCopyright("Copyright © 2015 Babbington Slade")] 9 | 10 | [assembly: ComVisible(false)] 11 | [assembly: Guid("0EB1C849-A6D0-448B-A2FD-DCC7338D8FF7")] 12 | 13 | [assembly: AssemblyVersion("1.0.0.0")] 14 | [assembly: AssemblyFileVersion("1.0.0.0")] 15 | 16 | -------------------------------------------------------------------------------- /src/bin/Debug/SSAddin.dna: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/Packages/Excel-DNA.0.30.3/content/ExcelDna-Template.dna: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 20 | -------------------------------------------------------------------------------- /src/Packages/Excel-DNA.0.32.0/content/ExcelDna-Template.dna: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 20 | -------------------------------------------------------------------------------- /src/Packages/Packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /cfg/SSAddin.xll.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/SSAddin.xll.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/ssaddin.dna: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/AddIn.cs: -------------------------------------------------------------------------------- 1 | // Copyright Babbington Slade Ltd 2 | using System; 3 | using System.Diagnostics; 4 | using System.Net.PeerToPeer; 5 | using ExcelDna.Integration; 6 | 7 | namespace SSAddin { 8 | public class AddIn : IExcelAddIn { 9 | private static TraceSource m_TraceSource = new TraceSource( "ssaddin"); 10 | 11 | // We don't use System.Net.PeerToPeer.PeerName anywhere in the SSAddin code. 12 | // This is here to force loading of System.Net.dll 4.0.0.0 before Google.Apis.dll 13 | // asks for v2.0.5.0 as described in the link below. For some reason assembly 14 | // bindingRedirects in SSAddin.xll.config don't work for System.Net.dll, even 15 | // though they do work for System.Net.Http.Primitives. I don't want to have 16 | // to require the KB2468871 fix, which wouldn't install on my Win8 laptop 17 | // anyway. One more thing: if you're trying to do bindingRedirects in your 18 | // .xll.config, do make sure ExcelDnaPack isn't packing it into the XLL. It 19 | // will if it find a .xll.config with a matching base name next to the .dna 20 | // file. JOS 2017-05-06 21 | // https://github.com/google/google-api-dotnet-client/issues/378 22 | // http://blog.slaks.net/2013-12-25/redirecting-assembly-loads-at-runtime/ 23 | private static PeerName m_PeerName = new PeerName( "dummy" ); 24 | 25 | public void AutoOpen( ) { 26 | Logr.Log( "AutoOpen" ); 27 | ExcelIntegration.RegisterUnhandledExceptionHandler( e => "EXCEPTION: " + (e as Exception).Message); 28 | } 29 | 30 | public void AutoClose( ) { 31 | Logr.Log( "AutoClose" ); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/Logr.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | 8 | namespace SSAddin { 9 | class Logr { 10 | 11 | protected static TextWriterTraceListener m_TextOut; 12 | 13 | static Logr( ) { 14 | // Trace goes to /dev/null in Release mode unless some kind of debug tool is used. 15 | // Need to add a listener to send to localFS. JOS 2015-05-19 16 | string logpath = String.Format( "{0}\\ssaddin_{1}.log", System.IO.Path.GetTempPath( ), Process.GetCurrentProcess( ).Id); 17 | m_TextOut = new TextWriterTraceListener( System.IO.File.CreateText( logpath)); 18 | Trace.Listeners.Add( m_TextOut); 19 | } 20 | 21 | public static void Log( string ln ) { 22 | // Add code here to add threadId, processId and timestamp 23 | // Why? https://msdn.microsoft.com/en-us/library/system.diagnostics.tracelistener.traceoutputoptions%28v=vs.100%29.aspx 24 | // Yes: TextWriterTraceListener WriteLine ignores the TraceOptions that add ThreadId, ProcessId etc 25 | // And I can't get FileLogTraceListener to add those either. Can I be arsed with all the TraceEvent crap necessary? 26 | // No! I'm writing multi threaded code, so I want my log lines to have threadIds without any explicit code on my part! 27 | // Is that too much to ask? 28 | string tstamp = DateTime.Now.ToString( "o"); 29 | int tid = Thread.CurrentThread.ManagedThreadId; 30 | int pid = Process.GetCurrentProcess( ).Id; 31 | Trace.WriteLine( String.Format( "{0} {1} {2} {3}", tstamp, pid, tid, ln )); 32 | // In release mode writes are buffered, but we want to see them logged immediately, so flush. 33 | Trace.Flush( ); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SSAddin 2 | SSAddin, the SpreadServe Addin is a conventional Excel XLL addin implemented in C#. It has no build or run time dependencies on the [SpreadServe](http://spreadserve.com>) server runtime, and can be used independently in a regular desktop Excel installation, or in [SpreadServe](http://spreadserve.com>) itself. SSAddin supports quandl, cron style scheduled execution, and web socket live updates. SSAddin is freely available under the Apache License 2.0 3 | 4 | ## Binaries 5 | You can download ready to install 32 & 64 bit binaries from [SpreadServe's download page](http://spreadserve.com/s3/downloads.html). 6 | 7 | ## Google Analytics 8 | SSAddin gives you access to Google Analytics Reporting v3 API via the s2ganalytics and s2gacache worksheet functions. See the ganalytics_sessions2.xls worksheet for an example. 9 | 10 | ## Baremetrics 11 | SSAddin gives you access to the Baremetrics API via the s2baremetrics and s2bcache worksheet functions. See the baremetrics_summary1.xlsx worksheet for an example. 12 | 13 | ## Quandl 14 | quandl.com already distributes a perfectly good Excel addin, so how is SSAddin different? SSAddin uses no VBA and no GUI. It has no menu cluttering your Excel menu bar, no dialog or message boxes popping up. Everything is achieved via worksheet functions, and all network round trips are handled on a background thread so your Excel UI never blocks waiting for data to download from quandl.com. 15 | 16 | ## Tiingo 17 | Tiingo is an exciting new financial data portal challenging high priced incumbents like Bloomberg and Thomson Reuters. Recently tiingo.com has added API access to historical data, which is now supported by SSAddin. 18 | 19 | ## Cron 20 | SSAddin enables the creation of cron style timer jobs in Excel that trigger RTD updates on the schedule you specify. Cron timers can be used to trigger recalculations, or to launch scheduled downloads from quandl.com 21 | 22 | ## Today 23 | SSAddin provides the s2today function. s2today is a non volatile eqivalent of Excel's TODAY. It enables invocation in spreadsheets using RTD without triggering endless calc cycles. 24 | 25 | ## Web sockets 26 | SSAddin supports subscription to live ticking web data via web sockets. 27 | 28 | ## Documentation 29 | http://spreadserve-addin.readthedocs.org/en/latest/index.html 30 | 31 | ## Acknowledgements 32 | SSAddin builds on several other excellent OSS projects: Excel-DNA, NCrontab, WebSockets4Net and JSON.NET. We use a modified NCrontab that extends Unix style cron schedules to allow more finegrained timing specifications using seconds as well as minutes, hours, days and days of the week. 33 | 34 | ## Contact 35 | john dot osullivan at spreadserve dot com 36 | 37 | -------------------------------------------------------------------------------- /src/TiingoRealTimeMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using WebSocket4Net; 6 | 7 | namespace SSAddin { 8 | class TiingoRealTimeMessageHandler { 9 | 10 | public delegate void MktDataTick( IList tick); 11 | public delegate void HeartBeat(int count); 12 | public delegate void SetSubID( string subID); 13 | 14 | protected static DataCache s_Cache = DataCache.Instance( ); 15 | 16 | protected MktDataTick m_Tick; 17 | protected HeartBeat m_HB; 18 | protected SetSubID m_SetSubID; 19 | protected WebSocket m_Socket; 20 | protected int m_HBCount; // tiingo websock heartbeat count 21 | 22 | #region Worker thread 23 | 24 | public TiingoRealTimeMessageHandler( WebSocket ws, MktDataTick tick, HeartBeat hb, SetSubID ssid) { 25 | // RTDServer.GetInstance( ) can instance the RTD server and all the RTD 26 | // COM machinery the first time we call it so we won't init as a static. 27 | // TiingoRealTimeMessageHandler and TWSCallback only get instanced when 28 | // a worksheet has called twebsock, so we know it's OK to instance 29 | // RTDServer at this point as the user definitely wants to do RT stuff. 30 | m_Tick = tick; 31 | m_HB = hb; 32 | m_Socket = ws; 33 | m_HBCount = 0; 34 | m_SetSubID = ssid; 35 | } 36 | 37 | public void MessageReceived(IDictionary msg) { 38 | if (msg == null) { 39 | Logr.Log(String.Format("TiingoRealTimeMessageHandler.MessageReceived: null msg!")); 40 | return; 41 | } 42 | if (!msg.ContainsKey("messageType")) { 43 | Logr.Log(String.Format("TiingoRealTimeMessageHandler.MessageReceived: missing messageType field!")); 44 | return; 45 | } 46 | // for example messages https://api.tiingo.com/docs/iex/realtime#priceData 47 | string mt = msg["messageType"].ToString( ); 48 | switch (mt) { 49 | case "I": // Informational 50 | if (msg.ContainsKey( "data" )) { 51 | var dd = (IDictionary)msg["data"]; 52 | if (dd.ContainsKey( "subscriptionId" )) { 53 | m_SetSubID( dd["subscriptionId"].ToString( ) ); 54 | } 55 | } 56 | break; 57 | case "H": // Heartbeat 58 | m_HB( ++m_HBCount); 59 | break; 60 | case "A": // Market data 61 | 62 | if (msg.ContainsKey("data")) { 63 | m_Tick( (IList)msg["data"]); 64 | } 65 | break; 66 | case "E": // Error 67 | break; 68 | default: 69 | Logr.Log( String.Format("TiingoRealTimeMessageHandler.MessageReceived: unexpected messageType({0})!", mt)); 70 | break; 71 | } 72 | } 73 | 74 | #endregion Worker thread 75 | 76 | #region Pool thread 77 | 78 | 79 | 80 | #endregion Pool thread 81 | 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /docs/install.rst: -------------------------------------------------------------------------------- 1 | Installing The SpreadServe Addin 2 | ================================ 3 | 4 | **Installing the addin for the first time** 5 | 6 | * Get the XLL from http://spreadserve.com/s3/downloads.html or source from https://github.com/SpreadServe/SSAddin 7 | * Install SSAddin.xll as an Excel addin. 8 | 9 | * Use SSAddin64.xll if you're running a 64 bit Excel. 10 | * Watch this video if you're unsure about adding an addin https://www.youtube.com/watch?v=i_sijj1NZFM 11 | * Put SSAddin.xll.config in the same directory as SSAddin.xll, and edit it to add Tiingo, Quandl and Baremetrics keys if you use those services. 12 | 13 | * Create a new sheet, or load one of the test sheets to check that the addin is loaded. 14 | 15 | * Hit *fx* on the formula bar to get the Insert Function dialog. 16 | * Select the SpreadServe Addin function category. 17 | * You should see `s2cron`, `s2quandl` and other SpreadServe Addin functions listed. 18 | 19 | * Bear in mind that the SpreadServe Addin does not add a ribbon menu. It's designed to work entirely 20 | through worksheet functions. 21 | 22 | **SpreadServe Addin test sheets** 23 | 24 | There are several test spreadsheets in the zip in the xls directory. These sheets all use RTD updates, 25 | so make sure you are in automatic calculation mode. Go to ``Formulas/Calculation Options`` in Excel and 26 | select Automatic. Use ctrl-alt-F9 to recalc everything and force the RTD subscriptions through. 27 | 28 | * ``cron1.xls``: demonstrates the use of the ``s2cron`` and ``s2sub`` functions to set up and track a timer 29 | that goes off every 20 seconds. The timer will stop at the end of the day. 30 | * ``cron2.xls``: uses of the ``s2cron`` and ``s2sub`` functions to set up and track a timer 31 | that goes off every 5 seconds. Note how the start and end dates are set in the s2cfg sheet so the 32 | timer will run beyond the end of the day, for as many days as the sheet is running. 33 | * ``cron3.xls``: uses of the ``s2cron`` and ``s2sub`` functions to set up and track a timer 34 | that goes off daily at 1430. Note how the start and end dates are set in the s2cfg sheet. 35 | * ``quandl1.xls``: uses ``s2quandl`` to launch a quandl query on a background thread in the subs sheet, 36 | and ``s2cache`` to pull the query result set into cells on the data sheet. You may have to ctrl-alt-F9 37 | a second time to force ``s2qcache`` execution in the subs sheet. 38 | * ``quandl2.xls``: a variation on quandl1. The two differences are the offsetting of the result set in 39 | the ``data`` sheet, and the use of ``s2vqcache`` instead of ``s2qcache``. The offsetting allows result sets 40 | to appear anywhere in a sheet instead of being anchored to the top left cell. ``s2vqcache`` is a volatile 41 | version of ``s2qcache``. Use of the volatile function avoids the need for a second ctrl-alt-F9. 42 | * ``quandl3.xls``: combines the cron and quandl features to implement a quandl query that is executed every 43 | 30 seconds. 44 | * ``wsock.xls``: uses the `s2websock` function to subscribe to updates from an automated sheet hosted 45 | by SpreadServe. 46 | * ``tiingows1.xls``: uses the `s2twebsock` and `s2sub` functions to subscribe to live ticking IEX market data 47 | from Tiingo. NB you will need to put your Tiingo authorization token into the s2cfg sheet to connect to Tiingo, 48 | and you'll need to be permissioned for IEX data at Tiingo. 49 | * ``tiingows_option1.cls``: using `s2twebsock` and `s2sub` to drive a Black Scholes option calc with ticking 50 | IEX market data. 51 | 52 | Some of the example sheets have ``_proxy`` suffixed to the name. These alternate versions are designed to work 53 | from behind an internet proxy. They have extra config sheet entries to configure username, password and proxy 54 | connection details. If you're in a corporate environment you'll probably need to use these. 55 | -------------------------------------------------------------------------------- /src/ProxyConnectorBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using System.Net; 4 | using System.Net.Sockets; 5 | using SuperSocket.ClientEngine; 6 | 7 | namespace SSAddin 8 | { 9 | public abstract class ProxyConnectorBase : IProxyConnector 10 | { 11 | public EndPoint ProxyEndPoint { get; private set; } 12 | 13 | protected static Encoding ASCIIEncoding = new ASCIIEncoding(); 14 | 15 | #if SILVERLIGHT && !WINDOWS_PHONE 16 | protected SocketClientAccessPolicyProtocol ClientAccessPolicyProtocol { get; private set; } 17 | 18 | public ProxyConnectorBase(EndPoint proxyEndPoint, SocketClientAccessPolicyProtocol clientAccessPolicyProtocol) 19 | { 20 | ProxyEndPoint = proxyEndPoint; 21 | ClientAccessPolicyProtocol = clientAccessPolicyProtocol; 22 | } 23 | 24 | #else 25 | public ProxyConnectorBase(EndPoint proxyEndPoint) 26 | { 27 | ProxyEndPoint = proxyEndPoint; 28 | } 29 | #endif 30 | 31 | public abstract void Connect(EndPoint remoteEndPoint); 32 | 33 | private EventHandler m_Completed; 34 | 35 | public event EventHandler Completed 36 | { 37 | add { m_Completed += value; } 38 | remove { m_Completed -= value; } 39 | } 40 | 41 | protected void OnCompleted(ProxyEventArgs args) 42 | { 43 | if (m_Completed == null) 44 | return; 45 | 46 | m_Completed(this, args); 47 | } 48 | 49 | protected void OnException(Exception exception) 50 | { 51 | OnCompleted(new ProxyEventArgs(exception)); 52 | } 53 | 54 | protected void OnException(string exception) 55 | { 56 | OnCompleted(new ProxyEventArgs(new Exception(exception))); 57 | } 58 | 59 | protected bool ValidateAsyncResult(SocketAsyncEventArgs e) 60 | { 61 | if (e.SocketError != SocketError.Success) 62 | { 63 | var socketException = new SocketException((int)e.SocketError); 64 | OnCompleted(new ProxyEventArgs(new Exception(socketException.Message, socketException))); 65 | return false; 66 | } 67 | 68 | return true; 69 | } 70 | 71 | protected void AsyncEventArgsCompleted(object sender, SocketAsyncEventArgs e) 72 | { 73 | if (e.LastOperation == SocketAsyncOperation.Send) 74 | ProcessSend(e); 75 | else 76 | ProcessReceive(e); 77 | } 78 | 79 | protected void StartSend(Socket socket, SocketAsyncEventArgs e) 80 | { 81 | bool raiseEvent = false; 82 | 83 | try 84 | { 85 | raiseEvent = socket.SendAsync(e); 86 | } 87 | catch (Exception exc) 88 | { 89 | OnException(new Exception(exc.Message, exc)); 90 | return; 91 | } 92 | 93 | if (!raiseEvent) 94 | { 95 | ProcessSend(e); 96 | } 97 | } 98 | 99 | protected virtual void StartReceive(Socket socket, SocketAsyncEventArgs e) 100 | { 101 | bool raiseEvent = false; 102 | 103 | try 104 | { 105 | raiseEvent = socket.ReceiveAsync(e); 106 | } 107 | catch (Exception exc) 108 | { 109 | OnException(new Exception(exc.Message, exc)); 110 | return; 111 | } 112 | 113 | if (!raiseEvent) 114 | { 115 | ProcessReceive(e); 116 | } 117 | } 118 | 119 | protected abstract void ProcessConnect(Socket socket, object targetEndPoint, SocketAsyncEventArgs e); 120 | 121 | protected abstract void ProcessSend(SocketAsyncEventArgs e); 122 | 123 | protected abstract void ProcessReceive(SocketAsyncEventArgs e); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/WSCallback.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using WebSocket4Net; 6 | using Newtonsoft.Json; 7 | 8 | namespace SSAddin { 9 | #region Deserialization classes 10 | // rtwebsvr websock format 11 | class SSWebCell { 12 | public String id { get; set; } 13 | public String body { get; set; } 14 | } 15 | #endregion 16 | 17 | class WSCallback { 18 | public delegate void ClosedCB( string wskey ); 19 | 20 | protected string m_Key; 21 | protected WebSocket m_Client; 22 | protected ClosedCB m_ClosedCB; 23 | protected static DataCache s_Cache = DataCache.Instance( ); 24 | 25 | #region Worker thread 26 | 27 | public WSCallback( Dictionary work, ClosedCB ccb ) { 28 | // ctor will be invoked on the SSWebClient worker thread 29 | m_ClosedCB = ccb; 30 | m_Key = work["key"]; 31 | string url = work["url"]; 32 | try { 33 | m_Client = new WebSocket( url ); 34 | m_Client.Opened += new EventHandler( Opened ); 35 | m_Client.Error += new EventHandler( Error ); 36 | m_Client.Closed += new EventHandler( Closed ); 37 | m_Client.MessageReceived += new EventHandler( MessageReceived ); 38 | m_Client.DataReceived += new EventHandler( DataReceived ); 39 | m_Client.Open( ); 40 | } 41 | catch (System.ArgumentException ae) { 42 | Logr.Log( String.Format( "WSCallback.ctor: {0}", ae.Message ) ); 43 | } 44 | } 45 | 46 | #endregion Worker thread 47 | 48 | #region Pool thread 49 | 50 | // All the pool thread methods are callbacks that will be fire on 51 | // web socket events on pool threads. 52 | 53 | protected void UpdateRTD( string qkey, string subelem, string value ) { 54 | // The RTD server doesn't necessarily exist. If no cell calls 55 | // s2sub( ) it won't be instanced by Excel. 56 | RTDServer rtd = RTDServer.GetInstance( ); 57 | if (rtd == null) 58 | return; 59 | string stopic = String.Format( "websock.{0}.{1}", qkey, subelem ); 60 | rtd.CacheUpdate( stopic, value ); 61 | } 62 | 63 | void DataReceived( object sender, WebSocket4Net.DataReceivedEventArgs e ) { 64 | Logr.Log( String.Format( "DataReceived: {0}", e.Data ) ); 65 | } 66 | 67 | void MessageReceived( object sender, MessageReceivedEventArgs e ) { 68 | List updates = JsonConvert.DeserializeObject>( e.Message); 69 | Logr.Log( String.Format( "MessageReceived: updates.Count({0})", updates.Count ) ); 70 | if (updates.Count == 0) 71 | return; 72 | RTDServer rtd = RTDServer.GetInstance( ); 73 | if (rtd != null) { 74 | rtd.CacheUpdateBatch( String.Format( "websock.{0}", m_Key), updates); 75 | } 76 | s_Cache.UpdateWSCache( m_Key, updates ); 77 | } 78 | 79 | void Closed( object sender, EventArgs e ) { 80 | Logr.Log( String.Format( "Closed: wskey({0})", m_Key ) ); 81 | if (m_ClosedCB != null) 82 | m_ClosedCB( m_Key ); 83 | UpdateRTD( m_Key, "status", "closed" ); 84 | } 85 | 86 | void Error( object sender, SuperSocket.ClientEngine.ErrorEventArgs e ) { 87 | Logr.Log( String.Format( "Error: wskey({0}) {1}", m_Key, e.Exception.Message ) ); 88 | UpdateRTD( m_Key, "status", "error" ); 89 | UpdateRTD( m_Key, "error", e.Exception.Message ); 90 | } 91 | 92 | void Opened( object sender, EventArgs e ) { 93 | Logr.Log( String.Format( "Opened: wskey({0})", m_Key ) ); 94 | UpdateRTD( m_Key, "status", "open" ); 95 | } 96 | 97 | #endregion Pool thread 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/GoogleAnalyticsAPI.cs: -------------------------------------------------------------------------------- 1 | using Google.Apis.Analytics.v3; 2 | using Google.Apis.Analytics.v3.Data; 3 | using Google.Apis.Services; 4 | using System.Security.Cryptography.X509Certificates; 5 | using Google.Apis.Auth.OAuth2; 6 | using System.Collections.Generic; 7 | using System.Linq; 8 | using System; 9 | 10 | namespace SSAddin 11 | { 12 | // AnalyticDataPoint encapsulates a result set from Google Analytics 13 | public class AnalyticDataPoint { 14 | public AnalyticDataPoint( ) 15 | { 16 | Rows = new List>(); 17 | } 18 | public IList ColumnHeaders { get; set; } 19 | public List> Rows { get; set; } 20 | } 21 | 22 | public class GoogleAnalyticsAPI 23 | { 24 | protected static AnalyticsService s_Service { get; set; } 25 | protected static IList s_Profiles { get; set; } 26 | protected static GoogleAnalyticsAPI s_Instance; 27 | 28 | #region Worker thread methods 29 | 30 | // Yes, this singleton instance method is a bit clunky with it's parameters. Bear in mind 31 | // that keypath and email come from ConfigSheet, and all the ConfigSheet invocations happen 32 | // on the Excel thread. But all the GoogleAnalyticsAPI methods execute on the worker thread. 33 | // So to get rid of these parms we'd have to introduce locking like DataCache or SSWebClient. 34 | // From a threading perspective it's cleaner and simpler to pass keypath and email in from 35 | // every invocation of s2ganalytics on the Excel thread via the request dictionary and have 36 | // the worker thread fire this method after getting the incoming request. 37 | public static GoogleAnalyticsAPI Instance( string keypath, string email) 38 | { 39 | if (s_Instance == null) { 40 | Logr.Log(String.Format("GoogleAnalyticsAPI.Instance keypath({0}) email({1})", keypath, email)); 41 | s_Instance = new GoogleAnalyticsAPI( keypath, email ); 42 | var response = s_Service.Management.Profiles.List("~all", "~all").Execute(); 43 | s_Profiles = response.Items; 44 | } 45 | return s_Instance; 46 | } 47 | 48 | protected GoogleAnalyticsAPI(string keyPath, string accountEmailAddress) 49 | { 50 | // "notasecret" is the default password Google supplies when you generate a P12 for a service account 51 | var certificate = new X509Certificate2(keyPath, "notasecret", X509KeyStorageFlags.Exportable); 52 | var credentials = new ServiceAccountCredential( 53 | new ServiceAccountCredential.Initializer(accountEmailAddress) { 54 | Scopes = new[] { AnalyticsService.Scope.AnalyticsReadonly } 55 | }.FromCertificate(certificate)); 56 | s_Service = new AnalyticsService(new BaseClientService.Initializer() { 57 | HttpClientInitializer = credentials, 58 | ApplicationName = "WorthlessVariable" 59 | }); 60 | } 61 | 62 | 63 | public AnalyticDataPoint GetAnalyticsData( string dimensions, // comma separated list, no spaces 64 | string metrics, // comma separated list, no spaces 65 | string startDate, // yyyy-MM-dd, possibly supplied by s2today 66 | string endDate) // yyyy-MM-dd, possibly supplied by s2today 67 | { 68 | // TODO: add another SSAddin.xml.config key g2analytics.profile to select profile 69 | string profileId = s_Profiles[0].Id; 70 | if ( !profileId.Contains("ga:")) 71 | profileId = string.Format("ga:{0}", profileId); 72 | 73 | // Make initial call to service. Then check if a next link exists in the response, 74 | // if so parse and call again using start index param. 75 | GaData response = null; 76 | AnalyticDataPoint data = new AnalyticDataPoint(); 77 | do { 78 | int startIndex = 1; 79 | if (response != null && !string.IsNullOrEmpty(response.NextLink)) { 80 | Uri uri = new Uri(response.NextLink); 81 | var parameters = uri.Query.Split('&'); 82 | string s = parameters.First(i => i.Contains("start-index")).Split('=')[1]; 83 | startIndex = int.Parse(s); 84 | } 85 | DataResource.GaResource.GetRequest request = s_Service.Data.Ga.Get( profileId, startDate, endDate, metrics); 86 | request.Dimensions = dimensions; 87 | response = request.Execute(); 88 | data.ColumnHeaders = response.ColumnHeaders; 89 | data.Rows.AddRange(response.Rows); 90 | } while (!string.IsNullOrEmpty(response.NextLink)); 91 | return data; 92 | } 93 | 94 | #endregion 95 | } 96 | } -------------------------------------------------------------------------------- /src/RTDServer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Reflection; 6 | using System.Runtime.InteropServices; 7 | using ExcelDna.Integration; 8 | using ExcelDna.Integration.Rtd; 9 | 10 | namespace SSAddin { 11 | [ComVisible(true)] 12 | class RTDServer : ExcelRtdServer { 13 | 14 | // Use a static collection of instance refs as we can't use a singleton pattern here. 15 | // Excel fires the ctor for an RTDServer via COM. We'll track the instances, and 16 | // hand out refs to other classes as necessary. We shouldn't need to lock on this 17 | // object as all access should be on the Excel thread. 18 | static protected List s_Instances = new List( ); 19 | 20 | protected Dictionary m_Subscriptions; // only Excel thread should touch this 21 | protected Dictionary m_TopicMap; // Excel and worker thread touch this 22 | 23 | #region Excel thread 24 | 25 | public static RTDServer GetInstance( ) { 26 | // This should only be called by methods executing on the Excel thread as there's 27 | // no locking on s_Instances. 28 | if (s_Instances.Count == 0) 29 | return null; 30 | return s_Instances[0]; 31 | } 32 | 33 | public RTDServer( ) { 34 | Logr.Log( "~A RTDServer created" ); 35 | m_Subscriptions = new Dictionary( ); 36 | m_TopicMap = new Dictionary( ); 37 | s_Instances.Add( this ); 38 | } 39 | 40 | protected int GetTopicId( Topic topic ) { 41 | // Why doesn't the Topic class expose the topicId ? 42 | var topicType = typeof( Topic ); 43 | var topicId = topicType.GetField( "TopicId", BindingFlags.GetField | BindingFlags.Instance | BindingFlags.NonPublic ); 44 | if (topicId == null) 45 | return 0; 46 | int rv = (int)topicId.GetValue( topic ); 47 | return rv; 48 | } 49 | 50 | protected override bool ServerStart( ) { 51 | Logr.Log( "~A RTDServer.ServerStart" ); 52 | return true; 53 | } 54 | 55 | protected override void ServerTerminate( ) { 56 | Logr.Log( "~A RTDServer.ServerTerminate" ); 57 | // Clear down any running timers... 58 | CronManager.Instance( ).Clear( ); 59 | s_Instances.Remove( this ); 60 | } 61 | 62 | protected override object ConnectData( Topic topic, System.Collections.Generic.IList topicInfo, ref bool newValues ) { 63 | lock (m_TopicMap) { 64 | string stopic = topicInfo[0]; 65 | int topicId = GetTopicId( topic ); 66 | Logr.Log( String.Format( "~A ConnectData: {0} - {1}", topicId, stopic ) ); 67 | m_Subscriptions.Add( topic, stopic ); 68 | m_TopicMap.Add( stopic, topic ); 69 | return ExcelErrorUtil.ToComError( ExcelError.ExcelErrorNA ); 70 | } 71 | } 72 | 73 | protected override void DisconnectData( Topic topic ) { 74 | lock (m_TopicMap) { 75 | string stopic = m_Subscriptions[topic]; 76 | Logr.Log( String.Format( "~A DisconnectData: {0}", stopic ) ); 77 | m_Subscriptions.Remove( topic ); 78 | m_TopicMap.Remove( stopic ); 79 | } 80 | } 81 | 82 | #endregion 83 | 84 | #region Worker or pool thread 85 | 86 | public void CacheUpdate( string stopic, string value ) { 87 | lock (m_TopicMap) { 88 | Logr.Log( String.Format( "~A CacheUpdate topic({0}) val({1})", stopic, value ) ); 89 | if (m_TopicMap.ContainsKey( stopic )) { 90 | Topic topic = m_TopicMap[stopic]; 91 | topic.UpdateValue( value ); 92 | } 93 | else { 94 | Logr.Log( String.Format( "~A CacheUpdate UNKNOWN topic({0}) val({1})", stopic, value ) ); 95 | } 96 | } 97 | } 98 | 99 | public void CacheUpdateBatch( string stroot, List updates) { 100 | lock (m_TopicMap) { 101 | Logr.Log( String.Format( "~A CacheUpdateBatch {0} {1}", stroot, updates.Count)); 102 | foreach ( SSWebCell wc in updates) { 103 | String stopic = String.Format( "{0}.{1}", stroot, wc.id); 104 | if (m_TopicMap.ContainsKey( stopic )) { 105 | Topic topic = m_TopicMap[stopic]; 106 | topic.UpdateValue( wc.body); 107 | Logr.Log( String.Format( "RTDServer.CacheUpdateBatch: topic({0}) value({1})", stopic, wc.body)); 108 | } 109 | } 110 | } 111 | } 112 | 113 | #endregion 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/JsonToDictionary.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Newtonsoft.Json; 6 | 7 | namespace SSAddin { 8 | class JsonToDictionary : JsonConverter { 9 | 10 | public override void WriteJson( JsonWriter writer, object value, JsonSerializer serializer ) { 11 | } 12 | 13 | public override object ReadJson( JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer ) { 14 | return ReadValue( reader ); 15 | } 16 | 17 | public override bool CanConvert( Type objectType ) { 18 | return ( objectType == typeof( IDictionary ) ); 19 | } 20 | 21 | public override bool CanWrite { 22 | get { return false; } 23 | } 24 | 25 | private object ReadValue( JsonReader reader ) { 26 | while (reader.TokenType == JsonToken.Comment) { 27 | if (!reader.Read( )) 28 | throw JsonSerializationExceptionCreate( reader, "Unexpected end when reading IDictionary." ); 29 | } 30 | switch (reader.TokenType) { 31 | case JsonToken.StartObject: 32 | return ReadObject( reader ); 33 | case JsonToken.StartArray: 34 | return ReadList( reader ); 35 | default: 36 | if (IsPrimitiveToken( reader.TokenType )) 37 | return reader.Value; 38 | throw JsonSerializationExceptionCreate( reader, string.Format( "Unexpected token when converting IDictionary: {0}", reader.TokenType ) ); 39 | } 40 | } 41 | 42 | private object ReadList( JsonReader reader ) { 43 | List list = new List( ); 44 | while (reader.Read( )) { 45 | switch (reader.TokenType) { 46 | case JsonToken.Comment: 47 | break; 48 | default: 49 | object v = ReadValue( reader ); 50 | list.Add( v ); 51 | break; 52 | case JsonToken.EndArray: 53 | return list; 54 | } 55 | } 56 | throw JsonSerializationExceptionCreate( reader, "Unexpected end when reading IDictionary." ); 57 | } 58 | 59 | private object ReadObject( JsonReader reader ) { 60 | IDictionary dictionary = new Dictionary( ); 61 | while (reader.Read( )) { 62 | switch (reader.TokenType) { 63 | case JsonToken.PropertyName: 64 | string propertyName = reader.Value.ToString( ); 65 | if (!reader.Read( )) 66 | throw JsonSerializationExceptionCreate( reader, "Unexpected end when reading IDictionary." ); 67 | object v = ReadValue( reader ); 68 | dictionary[propertyName] = v; 69 | break; 70 | case JsonToken.Comment: 71 | break; 72 | case JsonToken.EndObject: 73 | return dictionary; 74 | } 75 | } 76 | throw JsonSerializationExceptionCreate( reader, "Unexpected end when reading IDictionary." ); 77 | } 78 | 79 | //based on internal Newtonsoft.Json.JsonReader.IsPrimitiveToken 80 | internal static bool IsPrimitiveToken( JsonToken token ) { 81 | switch (token) { 82 | case JsonToken.Integer: 83 | case JsonToken.Float: 84 | case JsonToken.String: 85 | case JsonToken.Boolean: 86 | case JsonToken.Undefined: 87 | case JsonToken.Null: 88 | case JsonToken.Date: 89 | case JsonToken.Bytes: 90 | return true; 91 | default: 92 | return false; 93 | } 94 | } 95 | 96 | // based on internal Newtonsoft.Json.JsonSerializationException.Create 97 | private static JsonSerializationException JsonSerializationExceptionCreate( JsonReader reader, string message, Exception ex = null ) { 98 | return JsonSerializationExceptionCreate( reader as IJsonLineInfo, reader.Path, message, ex ); 99 | } 100 | 101 | // based on internal Newtonsoft.Json.JsonSerializationException.Create 102 | private static JsonSerializationException JsonSerializationExceptionCreate( IJsonLineInfo lineInfo, string path, string message, Exception ex ) { 103 | message = JsonPositionFormatMessage( lineInfo, path, message ); 104 | return new JsonSerializationException( message, ex ); 105 | } 106 | 107 | // based on internal Newtonsoft.Json.JsonPosition.FormatMessage 108 | internal static string JsonPositionFormatMessage( IJsonLineInfo lineInfo, string path, string message ) { 109 | if (!message.EndsWith( Environment.NewLine )) { 110 | message = message.Trim( ); 111 | if (!message.EndsWith( ".", StringComparison.Ordinal )) 112 | message += "."; 113 | message += " "; 114 | } 115 | message += string.Format( "Path '{0}'", path ); 116 | if (lineInfo != null && lineInfo.HasLineInfo( )) 117 | message += string.Format( ", line {0}, position {1}", lineInfo.LineNumber, lineInfo.LinePosition ); 118 | message += "."; 119 | return message; 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /docs/config.rst: -------------------------------------------------------------------------------- 1 | SpreadServe Addin Configuration 2 | =============================== 3 | 4 | **Log files** 5 | 6 | The SSAddin creates log files in your ``%TEMP%`` directory. To find them do this in a DOS box:: 7 | 8 | echo %TEMP% 9 | cd %TEMP% 10 | dir ssaddin* /od 11 | dir *.csv 12 | 13 | Note that the process ID of the Excel instance hosting your SSAddin is embedded in the log file 14 | name. The log file captures all the RTD updates sent by the addin to the sheet, together with 15 | their values. It also logs the start and end of Quandl queries. The addin also dumps the result 16 | sets returned from Quandl into CSV files in the ``%TEMP%`` directory. The files are named 17 | ``_.csv``. 18 | 19 | **s2cfg sheet** 20 | 21 | Any spreadsheet that uses SSAddin must have a sheet called ``s2cfg``. The SSAddin worksheet 22 | functions get their configuration from the s2cfg sheet and will fail if it doesn't exist 23 | or if its contents are not correctly laid out. The log files should alert you if there's a 24 | problem in your s2cfg sheet. They are also a good way of checking that the addin has composed 25 | your quandl or tiingo queries as you expected. Bear in mind these points on how the addin 26 | scans the s2cfg sheet for configuration. Also check the example sheets in the xls sub directory 27 | for concrete illustrations of the guidelines below. 28 | 29 | * SSAddin scans the s2cfg sheet from the first row downwards. It will stop scanning when it 30 | finds a row with an empty cell in column A. This means you can't have spaces between your 31 | config. It must all be in a single contiguous block from row 1 downwards. 32 | * The value in column A must be ``quandl``, ``tiingo``, ``cron``, ``websock`` or ``twebsock``. 33 | * Depending on the value in column A there are different expectations for the values in 34 | column B onwards. 35 | 36 | * ``quandl``: column B should be ``query`` or ``config`` 37 | 38 | * ``query``: column C should be the unique QueryKey that's passed to the ``s2quandl`` 39 | function, column D should be ``dataset`` and column E should name a Quandl dataset 40 | eg ``FRED/DED1`` or ``OPEC/ORB``. Any further columns should give key value pairs 41 | to tacked on to the Quandl query URL after the ``?`` For instance column F could be 42 | ``rows`` and column G ``5`` so that ``?rows=5`` is appended to the URL query submitted 43 | to quandl. 44 | * ``config``: column pairs from C & D onwards are reserved for name value pairs that 45 | apply to all queries. Currently only ``auth_token`` is supported. If you put ``auth_token`` 46 | in column C, then put your actual key in column D for it to be added to all queries. 47 | However, we recommend you put your key in SSAddin.xll.config instead, so you don't 48 | indavertently share your key when sharing your spreadsheet. 49 | 50 | * ``tiingo``: column B should be ``query`` or ``config`` 51 | 52 | * ``query``: column C should be the unique QueryKey that's passed to the ``s2tiingo`` 53 | function, column D should be ``ticker`` and column E should be a ticker symbol 54 | eg ``msft`` or ``aapl``. The ticker symbol should be lower case. Column F should 55 | be ``root``, followed by ``daily`` or ``funds`` in column G. Column H is optional. 56 | If it's present it should be ``leaf`` and then column I should be ``prices``. If 57 | it's absent a tiingo query that gets meta data for the symbol will be dispatched. 58 | Finally, columns J, K, L & M can be used to specify startDate and endDate for 59 | historical price queries. 60 | * ``config``: column pairs from C & D onwards are reserved for name value pairs that 61 | apply to all queries or Tiingo web socket connections (see twebsock below). 62 | Supported config keys are... 63 | 64 | * ``auth_token``: put ``auth_token`` in column C, and your actual key in column D 65 | for it to be added to all queries or used by twebsock. 66 | * ``http_proxy_host``: if this appears in column C then column D should give a proxy 67 | hostname. SSAddin will then connect via the proxy rather than direct to the internet. 68 | * ``http_proxy_port``: port for the proxy connection. 69 | * ``http_proxy_user``: user name for the proxy connection. Often this is in DOMAIN\USER 70 | format for Windows Active Directory user IDs. 71 | * ``http_proxy_password``: password for the proxy connection. 72 | 73 | * ``baremetrics``: column B should be ``query`` or ``config`` 74 | 75 | * ``query``: column C should be the unique QueryKey that's passed to the ``s2baremetrics`` 76 | function, column D should be ``qtype`` and column E should be a ``summary``, ``plan`` 77 | or ``metric``. For a qtype of ``plan`` or ``metric`` you need a following key/value 78 | pair that specifies which metric. The key, in column F should be ``metric``, and 79 | then in column G you should specify ``mrr``, `arpu``, ``ltv`` or any of the available 80 | metrics. Columns J, K, L & M can be used to specify ``start_date`` and ``end_date`` 81 | with the date values in columns K and M supplied by ``s2today`` or handcoded yyy-MM-dd 82 | strings. Don't use Excel's own `TODAY` function for these as it's volatile and will 83 | cause an endless calc cycle. Finally, if you're testing against the Sandbox API 84 | put ``sandbox`` in column L and ``TRUE`` in column M and ensure you have your 85 | Sandbox API key in SSAddin.xll.config. Don't forget to remove ``sandbox:TRUE`` and switch 86 | the API key in SSAddin.xll.config when you've finished testing! 87 | * ``config``: column pairs from C & D onwards are reserved HTTP proxy settings. See details 88 | for ``tiingo`` above. 89 | 90 | * ``twebsock``: when column B contains ``tiingo`` then column C specifies a ``SockKey`` to pass 91 | to ``s2twebsock``. Column D should give the URL for the Tiingo API socket eg ``wss://api.tiingo.com/iex`` 92 | 93 | * ``cron``: when column B contains ``tab`` then column C should have a unique ``CronKey`` 94 | that will be passed to the ``s2cron`` worksheet function which will then get the cron 95 | job specification from columns D to K. This job spec is then passed to SSAddin's internal 96 | `NCrontab `_ implementation. 97 | Bear in mind that SSAddin uses a hacked version of NCrontab that extends the spec to 98 | add seconds. 99 | 100 | * ``D``: seconds 101 | * ``E``: minutes 102 | * ``F``: hours 103 | * ``G``: days 104 | * ``H``: month 105 | * ``I``: weekday 106 | * ``J``: start - defaults to the start of today, today being the day when the process started. 107 | * ``K``: end - defaults to the end of today 108 | 109 | * ``websock``: when column B contains ``url`` then column C specifies a ``SockKey`` to pass 110 | to ``s2websock``. Column D should give the hostname of a SpreadServe server, column E the 111 | port number, and column F the rest of the URL, often referred to as the path. 112 | 113 | Note that if column B has any other value than described above it will be ignored. One convention 114 | you'll see in the SSAddin example s2cfg sheets is ``comment`` occurring in column B so that the 115 | rest of the row can be used as headers to describe the real values below. 116 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=c:\python27\scripts\sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\SpreadServe.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\SpreadServe.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /src/ssaddin.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {02C0D694-B363-4DE1-ACC7-D66EB669378B} 8 | Library 9 | Properties 10 | SSAddin 11 | SSAddin 12 | v4.5 13 | 512 14 | 15 | 16 | C:\osullivj\src\ExcelDna\Source 17 | 18 | 19 | true 20 | full 21 | false 22 | bin\Debug\ 23 | DEBUG;TRACE 24 | prompt 25 | 4 26 | false 27 | x86 28 | 29 | 30 | pdbonly 31 | true 32 | bin\Release\ 33 | TRACE 34 | prompt 35 | 4 36 | false 37 | 38 | 39 | 40 | 41 | 42 | 43 | False 44 | ..\..\ExcelDna034\Source\ExcelDna.Integration\bin\Debug\ExcelDna.Integration.dll 45 | 46 | 47 | Packages\Google.Apis.1.9.0\lib\net40\Google.Apis.dll 48 | 49 | 50 | Packages\Google.Apis.Analytics.v3.1.9.0.1110\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Analytics.v3.dll 51 | 52 | 53 | Packages\Google.Apis.Auth.1.9.0\lib\net40\Google.Apis.Auth.dll 54 | 55 | 56 | Packages\Google.Apis.Auth.1.9.0\lib\net40\Google.Apis.Auth.PlatformServices.dll 57 | 58 | 59 | Packages\Google.Apis.Core.1.9.0\lib\portable-net40+sl50+win+wpa81+wp80\Google.Apis.Core.dll 60 | 61 | 62 | Packages\Google.Apis.1.9.0\lib\net40\Google.Apis.PlatformServices.dll 63 | 64 | 65 | 66 | Packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll 67 | 68 | 69 | Packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll 70 | 71 | 72 | Packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll 73 | 74 | 75 | packages\NCrontab.3.3.1\lib\net35\NCrontab.dll 76 | 77 | 78 | Packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll 79 | 80 | 81 | 82 | 83 | 84 | 85 | False 86 | Packages\Microsoft.Net.Http.2.2.22\lib\net40\System.Net.Http.dll 87 | 88 | 89 | Packages\Microsoft.Net.Http.2.2.22\lib\net40\System.Net.Http.Extensions.dll 90 | 91 | 92 | Packages\Microsoft.Net.Http.2.2.22\lib\net40\System.Net.Http.Primitives.dll 93 | 94 | 95 | False 96 | Packages\Microsoft.Net.Http.2.2.22\lib\net40\System.Net.Http.WebRequest.dll 97 | 98 | 99 | 100 | Packages\WebSocket4Net.0.14.1\lib\net40\WebSocket4Net.dll 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | PreserveNewest 124 | 125 | 126 | 127 | 128 | PreserveNewest 129 | Designer 130 | 131 | 132 | 133 | 134 | 135 | == Post build with ExcelDna 0.34.6 nuget pkgs 136 | echo Build 32 and 64 bit version. using ExcelDna.xll and ExcelDna64.xll 137 | copy "$(ProjectDir)\Packages\ExcelDna.AddIn.0.34.6\tools\ExcelDna.xll" "$(TargetDir)SSAddin.xll" 138 | "$(ProjectDir)\Packages\ExcelDna.AddIn.0.34.6\tools\ExcelDnaPack.exe" "$(TargetDir)SSAddin.dna" /O "$(TargetDir)SSAddin-Packed.xll" /Y 139 | copy "$(TargetDir)SSAddin-Packed.xll" "$(TargetDir)SSAddin.xll" /Y 140 | del "$(TargetDir)SSAddin-Packed.xll" 141 | 142 | copy "$(SolutionDir)SSAddin.dna" "$(TargetDir)SSAddin64.dna" 143 | copy "$(ProjectDir)\Packages\ExcelDna.AddIn.0.34.6\tools\ExcelDna64.xll" "$(TargetDir)SSAddin64.xll" 144 | "$(ProjectDir)\Packages\ExcelDna.AddIn.0.34.6\tools\ExcelDnaPack.exe" "$(TargetDir)SSAddin64.dna" /O "$(TargetDir)SSAddin-Packed64.xll" /Y 145 | copy "$(TargetDir)SSAddin-Packed64.xll" "$(TargetDir)SSAddin64.xll" /Y 146 | del "$(TargetDir)SSAddin-Packed64.xll" 147 | 148 | 149 | 156 | -------------------------------------------------------------------------------- /src/HttpConnectProxy.cs: -------------------------------------------------------------------------------- 1 | // Adapted from... 2 | // https://github.com/kerryjiang/SuperSocket.ClientEngine/edit/master/Proxy/HttpConnectProxy.cs 3 | // ...on this advice... 4 | // http://stackoverflow.com/questions/23024121/how-to-use-proxies-with-the-websocket4net-library 5 | // ...and this discussion for the details on proxy auth... 6 | // https://github.com/kerryjiang/WebSocket4Net/issues/34 7 | // ...adding these changes... 8 | // https://github.com/thomaslevesque/SuperSocket.ClientEngine/commit/18109a20bdbe9d83709e64b7b3c9e650e4af8a94#diff-c330ae7198a267cf63adc658ad7bf836 9 | 10 | using System; 11 | using System.IO; 12 | using System.Net; 13 | using System.Net.Sockets; 14 | using System.Text; 15 | using SuperSocket.ClientEngine; 16 | 17 | namespace SSAddin 18 | { 19 | public class HttpConnectProxy : ProxyConnectorBase 20 | { 21 | 22 | class ConnectContext 23 | { 24 | public Socket Socket { get; set; } 25 | public SearchMarkState SearchState { get; set; } 26 | } 27 | 28 | private const string m_RequestTemplate = "CONNECT {0}:{1} HTTP/1.1\r\nHost: {0}:{1}\r\nProxy-Connection: Keep-Alive\r\n{2}\r\n"; 29 | 30 | private const string m_ResponsePrefix = "HTTP/"; 31 | private const char m_Space = ' '; 32 | 33 | private static byte[] m_LineSeparator; 34 | 35 | static HttpConnectProxy() 36 | { 37 | m_LineSeparator = ASCIIEncoding.GetBytes("\r\n\r\n"); 38 | } 39 | 40 | private int m_ReceiveBufferSize; 41 | 42 | #if SILVERLIGHT && !WINDOWS_PHONE 43 | public HttpConnectProxy(EndPoint proxyEndPoint, SocketClientAccessPolicyProtocol clientAccessPolicyProtocol) 44 | : this(proxyEndPoint, clientAccessPolicyProtocol, 128) 45 | { 46 | 47 | } 48 | 49 | public HttpConnectProxy(EndPoint proxyEndPoint, SocketClientAccessPolicyProtocol clientAccessPolicyProtocol, int receiveBufferSize) 50 | : base(proxyEndPoint, clientAccessPolicyProtocol) 51 | { 52 | m_ReceiveBufferSize = receiveBufferSize; 53 | } 54 | #else 55 | public HttpConnectProxy(EndPoint proxyEndPoint) 56 | : this(proxyEndPoint, 128) 57 | { 58 | 59 | } 60 | 61 | public HttpConnectProxy(EndPoint proxyEndPoint, int receiveBufferSize) 62 | : base(proxyEndPoint) 63 | { 64 | m_ReceiveBufferSize = receiveBufferSize; 65 | } 66 | #endif 67 | 68 | public override void Connect(EndPoint remoteEndPoint) 69 | { 70 | if (remoteEndPoint == null) 71 | throw new ArgumentNullException("remoteEndPoint"); 72 | 73 | if (!(remoteEndPoint is IPEndPoint || remoteEndPoint is DnsEndPoint)) 74 | throw new ArgumentException("remoteEndPoint must be IPEndPoint or DnsEndPoint", "remoteEndPoint"); 75 | 76 | try 77 | { 78 | #if SILVERLIGHT && !WINDOWS_PHONE 79 | ProxyEndPoint.ConnectAsync(ClientAccessPolicyProtocol, ProcessConnect, remoteEndPoint); 80 | #elif WINDOWS_PHONE 81 | ProxyEndPoint.ConnectAsync(ProcessConnect, remoteEndPoint); 82 | #else 83 | ProxyEndPoint.ConnectAsync(ProcessConnect, remoteEndPoint); 84 | #endif 85 | } 86 | catch (Exception e) 87 | { 88 | OnException(new Exception("Failed to connect proxy server", e)); 89 | } 90 | } 91 | 92 | protected override void ProcessConnect(Socket socket, object targetEndPoint, SocketAsyncEventArgs e) 93 | { 94 | if (e != null) 95 | { 96 | if (!ValidateAsyncResult(e)) 97 | return; 98 | } 99 | 100 | if (socket == null) 101 | { 102 | OnException(new SocketException((int)SocketError.ConnectionAborted)); 103 | return; 104 | } 105 | 106 | if (e == null) 107 | e = new SocketAsyncEventArgs(); 108 | 109 | string request; 110 | string authorizationHeader = null; 111 | string auth = Authorization; 112 | if (auth != null) 113 | { 114 | authorizationHeader = string.Format("Proxy-Authorization: {0}\r\n", auth); 115 | } 116 | 117 | if (targetEndPoint is DnsEndPoint) 118 | { 119 | var targetDnsEndPoint = (DnsEndPoint)targetEndPoint; 120 | request = string.Format(m_RequestTemplate, targetDnsEndPoint.Host, targetDnsEndPoint.Port, authorizationHeader); 121 | } 122 | else 123 | { 124 | var targetIPEndPoint = (IPEndPoint)targetEndPoint; 125 | request = string.Format(m_RequestTemplate, targetIPEndPoint.Address, targetIPEndPoint.Port, authorizationHeader); 126 | } 127 | 128 | var requestData = ASCIIEncoding.GetBytes(request); 129 | 130 | e.Completed += AsyncEventArgsCompleted; 131 | e.UserToken = new ConnectContext { Socket = socket, SearchState = new SearchMarkState(m_LineSeparator) }; 132 | e.SetBuffer(requestData, 0, requestData.Length); 133 | 134 | StartSend(socket, e); 135 | } 136 | 137 | protected override void ProcessSend(SocketAsyncEventArgs e) 138 | { 139 | if (!ValidateAsyncResult(e)) 140 | return; 141 | 142 | var context = (ConnectContext)e.UserToken; 143 | 144 | var buffer = new byte[m_ReceiveBufferSize]; 145 | e.SetBuffer(buffer, 0, buffer.Length); 146 | 147 | StartReceive(context.Socket, e); 148 | } 149 | 150 | protected override void ProcessReceive(SocketAsyncEventArgs e) 151 | { 152 | if (!ValidateAsyncResult(e)) 153 | return; 154 | 155 | var context = (ConnectContext)e.UserToken; 156 | 157 | int prevMatched = context.SearchState.Matched; 158 | 159 | int result = e.Buffer.SearchMark(e.Offset, e.BytesTransferred, context.SearchState); 160 | 161 | if (result < 0) 162 | { 163 | int total = e.Offset + e.BytesTransferred; 164 | 165 | if(total >= m_ReceiveBufferSize) 166 | { 167 | OnException("receive buffer size has been exceeded"); 168 | return; 169 | } 170 | 171 | e.SetBuffer(total, m_ReceiveBufferSize - total); 172 | StartReceive(context.Socket, e); 173 | return; 174 | } 175 | 176 | int responseLength = prevMatched > 0 ? (e.Offset - prevMatched) : (e.Offset + result); 177 | 178 | if (e.Offset + e.BytesTransferred > responseLength + m_LineSeparator.Length) 179 | { 180 | OnException("protocol error: more data has been received"); 181 | return; 182 | } 183 | 184 | var lineReader = new StringReader(ASCIIEncoding.GetString(e.Buffer, 0, responseLength)); 185 | 186 | var line = lineReader.ReadLine(); 187 | 188 | if (string.IsNullOrEmpty(line)) 189 | { 190 | OnException("protocol error: invalid response"); 191 | return; 192 | } 193 | 194 | //HTTP/1.1 2** OK 195 | var pos = line.IndexOf(m_Space); 196 | 197 | if (pos <= 0 || line.Length <= (pos + 2)) 198 | { 199 | OnException("protocol error: invalid response"); 200 | return; 201 | } 202 | 203 | var httpProtocol = line.Substring(0, pos); 204 | 205 | if (!httpProtocol.StartsWith(m_ResponsePrefix)) 206 | { 207 | OnException("protocol error: invalid protocol"); 208 | return; 209 | } 210 | 211 | var statusPos = line.IndexOf(m_Space, pos + 1); 212 | 213 | if (statusPos < 0) 214 | { 215 | OnException("protocol error: invalid response"); 216 | return; 217 | } 218 | 219 | int statusCode; 220 | //Status code should be 2** 221 | if (!int.TryParse(line.Substring(pos + 1, statusPos - pos - 1), out statusCode) || (statusCode > 299 || statusCode < 200)) 222 | { 223 | OnException("the proxy server refused the connection"); 224 | return; 225 | } 226 | 227 | OnCompleted(new ProxyEventArgs(context.Socket)); 228 | } 229 | 230 | public string Authorization { get; set; } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /docs/functions.rst: -------------------------------------------------------------------------------- 1 | SpreadServe Addin Worksheet Functions 2 | ===================================== 3 | 4 | These are the functions you can invoke directly from cells in your spreadsheet. 5 | 6 | **s2about**: get version information. 7 | 8 | Parameters 9 | 10 | * None 11 | 12 | Return value: a string detailing the SpreadServe Addin version, and the version of Excel hosting the adding. 13 | 14 | **s2cron**: setup scheduled timer. 15 | 16 | Parameters 17 | 18 | * ``CronKey``: a value or cell reference evaluating to a string that matches a value in column C of 19 | the s2cfg sheet. The s2cfg row with the matching column C value will be used to specify a cron job. 20 | See the cron1, 2 or 3 example sheets. 21 | 22 | Return value: "OK" if the function succeeds, an Excel error otherwise. 23 | 24 | **s2quandl**: launch a quandl query. 25 | 26 | Parameters 27 | 28 | * ``QueryKey``: a value or cell reference evaluating to a string that matches a value in column C of 29 | the s2cfg sheet. The s2cfg row with the matching column C value will be used to specify a quandl query. 30 | See the quandl1, 2 or 3 example sheets. 31 | * ``Trigger``: an optional trigger. The value isn't used inside the function, but a change in the input can 32 | be used to force repeat execution. See the quandl3.xls sheet for an example of an s2quandl trigger parameter 33 | hooked up to s2cron output to rerun a query on a timed basis. 34 | 35 | Return value: "OK" if the function succeeds, an Excel error otherwise. 36 | 37 | **s2qcache**: get a value from a quandl query result set. The position of the cell invoking this function is used 38 | to figure out which cell to get from the result set. 39 | 40 | Parameters 41 | 42 | * ``QueryKey``: should match the QueryKey given to `s2quandl`. 43 | * ``XOffset``: defaults to 0. If the left hand side of the result grid on your sheet is not column A this should 44 | be the number of columns across. 45 | * ``YOffset``: defaults to 0. If the top row of the result grid on your sheet is not row 1 this should 46 | be the number of rows down. 47 | * ``Trigger``: an optional trigger. The value isn't used inside the function, but a change in the input can 48 | be used to force repeat execution. See the quandl3.xls sheet for an example of an s2quandl trigger parameter 49 | hooked up to s2cron output to rerun a query on a timed basis. 50 | 51 | Return value: a value from the result set, or #N/A. 52 | 53 | **s2vqcache**: a volatile version of s2qcache. 54 | 55 | Parameters 56 | 57 | * ``QueryKey``: should match the QueryKey given to `s2quandl`. 58 | * ``XOffset``: defaults to 0. If the left hand side of the result grid on your sheet is not column A this should 59 | be the number of columns across. 60 | * ``YOffset``: defaults to 0. If the top row of the result grid on your sheet is not row 1 this should 61 | be the number of rows down. 62 | 63 | Return value: a value from the result set, or #N/A. 64 | 65 | **s2tiingo**: launch a tiingo query. 66 | 67 | Parameters 68 | 69 | * ``QueryKey``: a value or cell reference evaluating to a string that matches a value in column C of 70 | the s2cfg sheet. The s2cfg row with the matching column C value will be used to specify a tiingo query. 71 | See the tiingo1 or 2 example sheets. 72 | * ``Trigger``: an optional trigger. The value isn't used inside the function, but a change in the input can 73 | be used to force repeat execution. 74 | 75 | Return value: "OK" if the function succeeds, an Excel error otherwise. 76 | 77 | **s2tcache**: get a value from a tiingo query result set. The position of the cell invoking this function is used 78 | to figure out which cell to get from the result set. 79 | 80 | Parameters 81 | 82 | * ``QueryKey``: should match the QueryKey given to `s2tiingo`. 83 | * ``XOffset``: defaults to 0. If the left hand side of the result grid on your sheet is not column A this should 84 | be the number of columns across. 85 | * ``YOffset``: defaults to 0. If the top row of the result grid on your sheet is not row 1 this should 86 | be the number of rows down. 87 | * ``Trigger``: an optional trigger. The value isn't used inside the function, but a change in the input can 88 | be used to force repeat execution. 89 | 90 | Return value: a value from the result set, or #N/A. 91 | 92 | **s2vtcache**: a volatile version of s2tcache. 93 | 94 | Parameters 95 | 96 | * ``QueryKey``: should match the QueryKey given to `s2tiingo`. 97 | * ``XOffset``: defaults to 0. If the left hand side of the result grid on your sheet is not column A this should 98 | be the number of columns across. 99 | * ``YOffset``: defaults to 0. If the top row of the result grid on your sheet is not row 1 this should 100 | be the number of rows down. 101 | 102 | Return value: a value from the result set, or #N/A. 103 | 104 | **s2baremetrics**: launch a Baremetrics metric query. 105 | 106 | Parameters 107 | 108 | * ``QueryKey``: a value or cell reference evaluating to a string that matches a value in column C of 109 | the s2cfg sheet. The s2cfg row with the matching column C value will be used to specify a Baremetrics query. 110 | See the baremetrics_summary1 or baremetrics_metric1 example sheets. 111 | * ``Trigger``: an optional trigger. The value isn't used inside the function, but a change in the input can 112 | be used to force repeat execution. 113 | 114 | Return value: "OK" if the function succeeds, an Excel error otherwise. 115 | 116 | **s2bcache**: get a value from a Baremetrics query result set. 117 | 118 | Parameters 119 | 120 | * ``QueryKey``: should match the QueryKey given to **s2baremetrics**. 121 | * ``Date``: Baremetrics result sets are keyed on date; think of date as picking out a row. You should supply a 122 | string in ``yyyy-MM-dd`` format, or use the ``s2today`` function. Don't use Excel's volatile ``TODAY`` function as 123 | you'll cause an endless recalc cycle. 124 | * ``Field``: pick out a column in the result set row selected by ``Date``. 125 | * ``Trigger``: an optional trigger. The value isn't used inside the function, but a change in the input can 126 | be used to force repeat execution. 127 | 128 | Return value: a value from the result set, or #N/A. 129 | 130 | **s2vbcache**: a volatile version of **s2bcache**. 131 | 132 | Parameters 133 | 134 | * ``QueryKey``: should match the QueryKey given to **s2baremetrics**. 135 | * ``Date``: Baremetrics result sets are keyed on date; think of date as picking out a row. You should supply a 136 | string in ``yyyy-MM-dd`` format, or use the **s2today** function. Don't use Excel's volatile ``TODAY`` function as 137 | you'll cause an endless recalc cycle. 138 | * ``Field``: pick out a column in the result set row selected by ``Date``. 139 | 140 | Return value: a value from the result set, or #N/A. 141 | 142 | **s2sub**: subscribe to RTD updates generated by s2cron, s2quandl or s2websock. 143 | 144 | Parameters 145 | 146 | * ``SubCache``: [quandl|cron|websock] 147 | * ``CacheKey``: should match the CronKey or QueryKey given to s2cron or s2quandl. 148 | * ``Property``: [status|count|next|last|mX_Y_Z] count: cron event count for s2cron, rows in result set for s2quandl. 149 | next: time of next cron event. last: time of last cron event. 150 | 151 | Return value: RTD value, or #N/A. 152 | 153 | **s2websock**: subscribe via WebSockets to a page in a SpreadServe hosted sheet. 154 | 155 | Parameters 156 | 157 | * ``SockKey``: a value or cell reference evaluating to a string that matches a value in column C of 158 | the s2cfg sheet. The s2cfg row with the matching column C value will be used to specify the URL of 159 | a page in a SpreadServe hosted spreadsheet. See the websock1 example sheet. 160 | 161 | Return value: "OK" if the function succeeds, an Excel error otherwise. 162 | 163 | **s2twebsock**: subscribe via WebSockets to a Tiingo market data feed. 164 | 165 | Parameters 166 | 167 | * ``SockKey``: a value or cell reference evaluating to a string that matches a value in column C of 168 | the s2cfg sheet. The s2cfg row with the matching column C value will be used to specify the URL 169 | for the Tiingo websocket connection. See the tiingows1 example sheet. 170 | 171 | Return value: "OK" if the function succeeds, an Excel error otherwise. 172 | 173 | **s2wscache**: get a value from a WebSocket subscription cache. 174 | 175 | Parameters 176 | 177 | * ``SockKey``: should match the SockKey given to `s2websocket`. 178 | * ``CellKey``: for instance, m2_6_0 for col 3, row 7 on first sheet. Use 'Page Source' in your browser to 179 | examine the HTML on a page you want to subscribe to, and look for the div id tags to figure out the 180 | value you need. 181 | * ``Trigger``: an optional trigger. 182 | 183 | Return value: a value from the cache, or #N/A. 184 | 185 | **s2vwscache**: a volatile version of ``s2wscache``. 186 | 187 | Parameters 188 | 189 | * ``SockKey``: should match the SockKey given to `s2websocket`. 190 | * ``CellKey``: for instance, m2_6_0 for col 3, row 7 on first sheet. Use 'Page Source' in your browser to 191 | examine the HTML on a page you want to subscribe to, and look for the div id tags to figure out the 192 | value you need. 193 | 194 | Return value: a value from the cache, or #N/A. 195 | 196 | **s2today**: non volatile alternative to Excel's `TODAY`. 197 | 198 | Parameters 199 | 200 | * ``Offset``: 0 to get today, -1 for yesterday, +1 for tomorrow, -7 for a week ago, +7 for a week from now. 201 | 202 | Return value: a yyyy-MM-dd formatted date string. 203 | 204 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # SpreadServe documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Aug 06 13:34:13 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | import shlex 18 | 19 | # If extensions (or modules to document with autodoc) are in another directory, 20 | # add these directories to sys.path here. If the directory is relative to the 21 | # documentation root, use os.path.abspath to make it absolute, like shown here. 22 | #sys.path.insert(0, os.path.abspath('.')) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | #needs_sphinx = '1.0' 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = [] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['_templates'] 36 | 37 | # The suffix(es) of source filenames. 38 | # You can specify multiple suffix as a list of string: 39 | # source_suffix = ['.rst', '.md'] 40 | source_suffix = '.rst' 41 | 42 | # The encoding of source files. 43 | #source_encoding = 'utf-8-sig' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = u'SpreadServe Addin' 50 | copyright = u'2015, John O\'Sullivan' 51 | author = u'John O\'Sullivan' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '0.1' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '0.1.0' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # There are two options for replacing |today|: either, you set today to some 70 | # non-false value, then it is used: 71 | #today = '' 72 | # Else, today_fmt is used as the format for a strftime call. 73 | #today_fmt = '%B %d, %Y' 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | exclude_patterns = ['_build'] 78 | 79 | # The reST default role (used for this markup: `text`) to use for all 80 | # documents. 81 | #default_role = None 82 | 83 | # If true, '()' will be appended to :func: etc. cross-reference text. 84 | #add_function_parentheses = True 85 | 86 | # If true, the current module name will be prepended to all description 87 | # unit titles (such as .. function::). 88 | #add_module_names = True 89 | 90 | # If true, sectionauthor and moduleauthor directives will be shown in the 91 | # output. They are ignored by default. 92 | #show_authors = False 93 | 94 | # The name of the Pygments (syntax highlighting) style to use. 95 | pygments_style = 'sphinx' 96 | 97 | # A list of ignored prefixes for module index sorting. 98 | #modindex_common_prefix = [] 99 | 100 | # If true, keep warnings as "system message" paragraphs in the built documents. 101 | #keep_warnings = False 102 | 103 | # If true, `todo` and `todoList` produce output, else they produce nothing. 104 | todo_include_todos = False 105 | 106 | 107 | # -- Options for HTML output ---------------------------------------------- 108 | 109 | # The theme to use for HTML and HTML Help pages. See the documentation for 110 | # a list of builtin themes. 111 | html_theme = 'sphinx_rtd_theme' 112 | 113 | # Theme options are theme-specific and customize the look and feel of a theme 114 | # further. For a list of options available for each theme, see the 115 | # documentation. 116 | #html_theme_options = {} 117 | 118 | # Add any paths that contain custom themes here, relative to this directory. 119 | #html_theme_path = [] 120 | 121 | # The name for this set of Sphinx documents. If None, it defaults to 122 | # " v documentation". 123 | #html_title = None 124 | 125 | # A shorter title for the navigation bar. Default is the same as html_title. 126 | #html_short_title = None 127 | 128 | # The name of an image file (relative to this directory) to place at the top 129 | # of the sidebar. 130 | #html_logo = None 131 | 132 | # The name of an image file (within the static path) to use as favicon of the 133 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 134 | # pixels large. 135 | #html_favicon = None 136 | 137 | # Add any paths that contain custom static files (such as style sheets) here, 138 | # relative to this directory. They are copied after the builtin static files, 139 | # so a file named "default.css" will overwrite the builtin "default.css". 140 | html_static_path = ['_static'] 141 | 142 | # Add any extra paths that contain custom files (such as robots.txt or 143 | # .htaccess) here, relative to this directory. These files are copied 144 | # directly to the root of the documentation. 145 | #html_extra_path = [] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 148 | # using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names to 159 | # template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_domain_indices = True 164 | 165 | # If false, no index is generated. 166 | #html_use_index = True 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 175 | #html_show_sphinx = True 176 | 177 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 178 | #html_show_copyright = True 179 | 180 | # If true, an OpenSearch description file will be output, and all pages will 181 | # contain a tag referring to it. The value of this option must be the 182 | # base URL from which the finished HTML is served. 183 | #html_use_opensearch = '' 184 | 185 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 186 | #html_file_suffix = None 187 | 188 | # Language to be used for generating the HTML full-text search index. 189 | # Sphinx supports the following languages: 190 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 191 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 192 | #html_search_language = 'en' 193 | 194 | # A dictionary with options for the search language support, empty by default. 195 | # Now only 'ja' uses this config value 196 | #html_search_options = {'type': 'default'} 197 | 198 | # The name of a javascript file (relative to the configuration directory) that 199 | # implements a search results scorer. If empty, the default will be used. 200 | #html_search_scorer = 'scorer.js' 201 | 202 | # Output file base name for HTML help builder. 203 | htmlhelp_basename = 'SpreadServedoc' 204 | 205 | # -- Options for LaTeX output --------------------------------------------- 206 | 207 | latex_elements = { 208 | # The paper size ('letterpaper' or 'a4paper'). 209 | #'papersize': 'letterpaper', 210 | 211 | # The font size ('10pt', '11pt' or '12pt'). 212 | #'pointsize': '10pt', 213 | 214 | # Additional stuff for the LaTeX preamble. 215 | #'preamble': '', 216 | 217 | # Latex figure (float) alignment 218 | #'figure_align': 'htbp', 219 | } 220 | 221 | # Grouping the document tree into LaTeX files. List of tuples 222 | # (source start file, target name, title, 223 | # author, documentclass [howto, manual, or own class]). 224 | latex_documents = [ 225 | (master_doc, 'SpreadServe.tex', u'SpreadServe Documentation', 226 | u'John O\'Sullivan', 'manual'), 227 | ] 228 | 229 | # The name of an image file (relative to this directory) to place at the top of 230 | # the title page. 231 | #latex_logo = None 232 | 233 | # For "manual" documents, if this is true, then toplevel headings are parts, 234 | # not chapters. 235 | #latex_use_parts = False 236 | 237 | # If true, show page references after internal links. 238 | #latex_show_pagerefs = False 239 | 240 | # If true, show URL addresses after external links. 241 | #latex_show_urls = False 242 | 243 | # Documents to append as an appendix to all manuals. 244 | #latex_appendices = [] 245 | 246 | # If false, no module index is generated. 247 | #latex_domain_indices = True 248 | 249 | 250 | # -- Options for manual page output --------------------------------------- 251 | 252 | # One entry per manual page. List of tuples 253 | # (source start file, name, description, authors, manual section). 254 | man_pages = [ 255 | (master_doc, 'spreadserve', u'SpreadServe Documentation', 256 | [author], 1) 257 | ] 258 | 259 | # If true, show URL addresses after external links. 260 | #man_show_urls = False 261 | 262 | 263 | # -- Options for Texinfo output ------------------------------------------- 264 | 265 | # Grouping the document tree into Texinfo files. List of tuples 266 | # (source start file, target name, title, author, 267 | # dir menu entry, description, category) 268 | texinfo_documents = [ 269 | (master_doc, 'SpreadServe', u'SpreadServe Documentation', 270 | author, 'SpreadServe', 'One line description of project.', 271 | 'Miscellaneous'), 272 | ] 273 | 274 | # Documents to append as an appendix to all manuals. 275 | #texinfo_appendices = [] 276 | 277 | # If false, no module index is generated. 278 | #texinfo_domain_indices = True 279 | 280 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 281 | #texinfo_show_urls = 'footnote' 282 | 283 | # If true, do not generate a @detailmenu in the "Top" node's menu. 284 | #texinfo_no_detailmenu = False 285 | -------------------------------------------------------------------------------- /src/DataCache.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reflection; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace SSAddin { 9 | // DataCache is the cache for non real time data like tiingo and quandl historical data result sets. 10 | // It also caches web socket data too. DataCache needs to be thread safe so worker threads can push 11 | // data in, and the Excel thread can pull data out. 12 | class DataCache { 13 | protected Dictionary> m_QCache = new Dictionary>( ); 14 | protected Dictionary m_GACache = new Dictionary(); 15 | protected Dictionary> m_BCache = new Dictionary>( ); 16 | protected Dictionary> m_THPCache = new Dictionary>( ); 17 | protected Dictionary m_WSCache = new Dictionary( ); 18 | protected static DataCache s_Instance; 19 | protected static object s_InstanceLock = new object( ); 20 | protected static char[] s_delimiters = { '.' }; 21 | 22 | private DataCache( ) { 23 | } 24 | 25 | public static DataCache Instance( ) { 26 | lock (s_InstanceLock) { 27 | if (s_Instance == null) { 28 | s_Instance = new DataCache( ); 29 | } 30 | return s_Instance; 31 | } 32 | } 33 | 34 | public int AddQuandlLine( string qkey, string[] line) { 35 | int count = 0; 36 | lock ( m_QCache) { 37 | if ( !m_QCache.ContainsKey( qkey)) 38 | m_QCache.Add( qkey, new List( )); 39 | List slist = m_QCache[qkey]; 40 | slist.Add( line ); 41 | count = slist.Count; 42 | } 43 | return count; 44 | } 45 | 46 | 47 | public void UpdateWSCache( string wkey, List updates) { 48 | lock (m_WSCache) { 49 | foreach (SSWebCell wc in updates) { 50 | string key = String.Format( "{0}.{1}", wkey, wc.id); 51 | m_WSCache[key] = wc.body; 52 | } 53 | } 54 | } 55 | 56 | public void UpdateTHPCache( string wkey, List updates ) { 57 | lock (m_THPCache) { 58 | m_THPCache[wkey] = updates; 59 | } 60 | } 61 | 62 | public void UpdateGACache(string wkey, AnalyticDataPoint updates) 63 | { 64 | lock (m_GACache) 65 | { 66 | m_GACache[wkey] = updates; 67 | } 68 | } 69 | 70 | public void UpdateBareCache(string wkey, dynamic updates) 71 | { 72 | if ( updates == null) { 73 | Logr.Log( String.Format( "UpdateBareCache wkey({0}) updates==null!", wkey ) ); 74 | return; 75 | } 76 | JToken metrics = null; 77 | if ( !updates.TryGetValue( "metrics", out metrics)) { 78 | Logr.Log( String.Format( "UpdateBareCache wkey({0}) couldn't get metrics!", wkey ) ); 79 | return; 80 | } 81 | lock (m_BCache) { 82 | // the object returned by baremetrics always has an element 83 | // called 'metrics', and metrics is always an array of objects 84 | // that have data and human_date attributes. Beyond that we 85 | // may have all sorts of other members and nesting. 86 | Dictionary cached = null; 87 | if (!m_BCache.TryGetValue( wkey, out cached )) 88 | cached = new Dictionary( ); 89 | foreach (JObject sum in metrics) { 90 | JToken sdate = null; 91 | if (sum.TryGetValue( "human_date", out sdate )) 92 | cached[sdate.ToString( )] = sum; 93 | } 94 | m_BCache[wkey] = cached; 95 | } 96 | } 97 | 98 | public bool ContainsQuandlKey( string qkey ) { 99 | lock (m_QCache) { 100 | return m_QCache.ContainsKey( qkey ); 101 | } 102 | } 103 | 104 | public void ClearQuandl( string qkey ) { 105 | lock (m_QCache) { 106 | if ( m_QCache.ContainsKey( qkey)) 107 | m_QCache.Remove( qkey ); 108 | } 109 | } 110 | 111 | public bool ContainsTiingoKey( string qkey ) { 112 | lock (m_THPCache) { 113 | return m_THPCache.ContainsKey( qkey ); 114 | } 115 | } 116 | 117 | public bool ContainsGAnalyticsKey(string qkey) 118 | { 119 | lock (m_GACache) { 120 | return m_GACache.ContainsKey(qkey); 121 | } 122 | } 123 | 124 | public bool ContainsBareKey( string qkey ) { 125 | lock (m_BCache) { 126 | return m_BCache.ContainsKey( qkey ); 127 | } 128 | } 129 | 130 | public void ClearTiingo( string qkey ) { 131 | lock (m_THPCache) { 132 | if (m_THPCache.ContainsKey( qkey )) 133 | m_THPCache.Remove( qkey ); 134 | } 135 | } 136 | 137 | public string GetQuandlCell( string qkey, int row, int col ) { 138 | lock (m_QCache) { 139 | if (!m_QCache.ContainsKey( qkey )) 140 | return null; 141 | List slist = m_QCache[qkey]; 142 | if (row >= slist.Count) 143 | return null; 144 | String[] sarray = slist.ElementAt( row ); 145 | if (col >= sarray.Length) 146 | return null; 147 | return sarray[col]; 148 | } 149 | } 150 | 151 | public string GetTiingoCell( string qkey, int row, int col ) { 152 | lock (m_THPCache) { 153 | if (!m_THPCache.ContainsKey( qkey )) 154 | return null; 155 | List thplist = m_THPCache[qkey]; 156 | if (row >= thplist.Count) 157 | return null; 158 | SSTiingoHistPrice thp = thplist.ElementAt( row ); 159 | PropertyInfo[] piarr = thp.GetType( ).GetProperties( ); 160 | // String[] sarray = slist.ElementAt( row ); 161 | if (col >= piarr.Length) 162 | return null; 163 | PropertyInfo pi = piarr[col]; 164 | return pi.GetValue( thp, BindingFlags.Default, null, null, null).ToString( ); 165 | } 166 | } 167 | 168 | public string GetGAnalyticsCell(string qkey, int row, int col, bool hdrs) 169 | { 170 | lock (m_GACache) { 171 | if (!m_GACache.ContainsKey(qkey)) 172 | return null; 173 | if ( col < 0 || row < 0) 174 | return null; 175 | AnalyticDataPoint adp = m_GACache[qkey]; 176 | // If headers is true, we consider the headers as row 0, and the 177 | // first data record is row 1 178 | if (hdrs) { 179 | if (row == 0) { 180 | if (col >= adp.ColumnHeaders.Count) 181 | return null; 182 | return adp.ColumnHeaders[col].Name; 183 | } 184 | // adjust row down: row==1 refers to row 0 in adp.Rows 185 | // when hdrs is true 186 | row = row - 1; 187 | } 188 | if (row >= adp.Rows.Count) 189 | return null; 190 | IList record = adp.Rows[row]; 191 | if (col >= record.Count) 192 | return null; 193 | return record[col]; 194 | } 195 | } 196 | 197 | public string GetWSCell( string wkey, string ckey) { 198 | string key = String.Format( "{0}.{1}", wkey, ckey ); 199 | lock (m_WSCache) { 200 | if (!m_WSCache.ContainsKey( key )) 201 | return null; 202 | return m_WSCache[key]; 203 | } 204 | } 205 | 206 | public string GetBareField( string qkey, string sdate, string field) { 207 | // TODO: add more logging for failed nav thru the obj tree 208 | lock (m_BCache) { 209 | if (!m_BCache.ContainsKey( qkey )) 210 | return null; 211 | Dictionary cached = m_BCache[qkey]; 212 | if (!cached.ContainsKey( sdate )) 213 | return null; 214 | JObject jobj = cached[sdate]; 215 | string[] subs = field.Split( s_delimiters ); 216 | JToken jsub = null; 217 | JArray jarr = null; 218 | int inx = 0; 219 | bool ok = false; 220 | foreach ( string sub in subs) { 221 | if ( jobj != null) { 222 | ok = jobj.TryGetValue( sub, out jsub ); 223 | } 224 | else if ( jarr != null) { 225 | ok = Int32.TryParse( sub, out inx ); 226 | if (ok && inx < jarr.Count) 227 | jsub = jarr[inx]; 228 | else 229 | ok = false; 230 | } 231 | if (!ok) 232 | return null; 233 | // Have we hit an atomic, or do we go round again? 234 | if ( jsub is JValue) 235 | return jsub.ToString( ); 236 | // We haven't hit an atomic, so it's either an object or an array/ 237 | jobj = null; 238 | jarr = null; 239 | ok = false; 240 | if (jsub is JObject) 241 | jobj = jsub as JObject; 242 | else if (jsub is JArray) 243 | jarr = jsub as JArray; 244 | } 245 | return null; 246 | } 247 | } 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /src/CronManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Globalization; 5 | using System.Text; 6 | using System.Timers; 7 | using NCrontab; 8 | 9 | namespace SSAddin { 10 | class CronManager { 11 | protected static object s_InstanceLock = new object( ); 12 | protected static CronManager s_Instance; 13 | 14 | protected class CronTimer { 15 | // Keep internal copies of ctor parms 16 | protected String m_Key; 17 | protected String m_Cronex; 18 | protected DateTime? m_Start; 19 | protected DateTime? m_End; 20 | // Working storage for the timer 21 | protected IEnumerable m_Schedule; 22 | protected IEnumerator m_Iterator; 23 | protected Timer m_Timer; 24 | protected int m_Count = 0; 25 | protected String m_LastEventTime = ""; 26 | protected String m_NextEventTime = ""; 27 | protected bool m_Closed = false; 28 | 29 | #region Excel thread 30 | 31 | // Only the Excel thread should touch m_Timer.Enabled. If a pool thread can flip the 32 | // state of Enabled we may get race conditions with timers being inadvertently 33 | // re-enabled after being switched off. 34 | 35 | public CronTimer( String ckey, String cronex, DateTime? start, DateTime? end ) { 36 | // Save setup parms so later invocations of AddCron can check whether we need 37 | // to create a new instance or not. 38 | m_Key = ckey; 39 | m_Cronex = cronex; 40 | m_Start = start; 41 | m_End = end; 42 | // Now do the real biz of setting up the timer. 43 | DateTime sStart = start ?? DateTime.Now; 44 | DateTime sEnd = end ?? new DateTime( sStart.Year, sStart.Month, sStart.Day, 23, 59, 59 ); 45 | CrontabSchedule schedule = CrontabSchedule.Parse( cronex ); 46 | m_Schedule = schedule.GetNextOccurrences( sStart, sEnd ); 47 | m_Iterator = m_Schedule.GetEnumerator( ); 48 | m_Timer = new System.Timers.Timer( ); 49 | m_Timer.Enabled = false; 50 | m_Timer.AutoReset = false; 51 | m_Timer.Elapsed += this.OnTimerEvent; 52 | ScheduleTimer( ); 53 | m_Timer.Enabled = true; 54 | } 55 | 56 | public void Close( ) { 57 | // There's a nasty bug in the Timer class: setting the interval schedules 58 | // a timer callback even if Enabled is set false. This means you can't 59 | // stop a timer by setting Enabled=false if there is another callback 60 | // scheduled, and that callback will set Interval! 61 | // https://evolpin.wordpress.com/2014/04/25/the-curious-case-of-system-timers-timer/ 62 | // So we set a flag to tell the ScheduleTimer( ) method not to touch 63 | // m_Timer.Interval. And then hopefully, the GC will do it's stuff... 64 | m_Closed = true; 65 | } 66 | 67 | public string Cronex { 68 | get { return m_Cronex; } 69 | } 70 | 71 | public DateTime? Start { 72 | get { return m_Start; } 73 | } 74 | 75 | public DateTime? End { 76 | get { return m_End; } 77 | } 78 | 79 | #endregion Excel thread 80 | 81 | #region Pool thread 82 | 83 | public bool ScheduleTimer( ) { 84 | if (m_Closed) { 85 | Logr.Log( String.Format( "ScheduleTimer: {0} is closed", m_Key ) ); 86 | return false; 87 | } 88 | // NB this method is called by the Excel thread in the first instance. Subsequent calls are 89 | // on a pool thread, as .Net dispatches timer callbacks on pool threads, unless we specify otherwise. 90 | // I'm not bothering with a lock because the first timer event isn't scheduled until we do 91 | // m_Timer.Enabled = true below, and we don't touch the iterator after that, so there's no 92 | // chance that we'll have two threads touching m_Iterator at the same time. 93 | 94 | // What if the next time we got from m_Iterator is already in the past? 95 | // If it is keep moving fwd til we get a time in the future. Bear in mind there's 96 | // an error condition where Current can appear to be in the future when it's not. 97 | // If DateTime.Now==2015-06-03T20:03:04.9998700, and Current==2015-06-03T20:03:05 98 | // then CompareTo will tell us that Current is in the future, when for our 99 | // purposes it's not. If ticks is -ve then Current is in the past. If Current is 100 | // a small +/-ve then it's the same as Now since our unit of granularity in the 101 | // cron sys is 1 sec. There are 10,000 ticks to the millisec, 10,000,000 to the sec. 102 | // So we'll look for Current to be 1,000,000 ticks later than Now before scheduling. 103 | // Which is +1,000,000. If ticks is -ve then Current is in the past. This check 104 | // should also prevent interval==0 below! 105 | long ticks = m_Iterator.Current.Ticks - DateTime.Now.Ticks; 106 | while ( ticks < 1000000) { 107 | if ( !m_Iterator.MoveNext( )) { 108 | Logr.Log( String.Format( "ScheduleTimer: {0} exhausted", m_Key ) ); 109 | return false; 110 | } 111 | ticks = m_Iterator.Current.Ticks - DateTime.Now.Ticks; 112 | } 113 | m_LastEventTime = DateTime.Now.ToString( ); 114 | m_NextEventTime = m_Iterator.Current.ToString( ); 115 | // Ticks is number of 100 nanos since 0001-01-01T00:00:00. Diff between now 116 | // and next event time 10K is the number of millisecs until the next cron event 117 | // for ckey. https://msdn.microsoft.com/en-us/library/system.datetime.ticks%28v=vs.100%29.aspx 118 | // 10,000 ticks in 1 millisec 119 | long interval = Math.Abs( ticks / 10000 ); 120 | if (interval == 0) { 121 | // Given the code above, this should not happen! 122 | Logr.Log( String.Format( "ScheduleTimer: ZERO interval! ckey({0}) Current({1}) Now({2})", 123 | m_Key, m_Iterator.Current, DateTime.Now ) ); 124 | return false; 125 | } 126 | m_Timer.Interval = interval; 127 | Logr.Log( String.Format( "ScheduleTimer: ckey({0}) Current({1}) Now({2})", m_Key, m_Iterator.Current, DateTime.Now ) ); 128 | return true; 129 | } 130 | 131 | protected void UpdateRTD( string qkey, string subelem, string value ) { 132 | // The RTD server doesn't necessarily exist. If no cell calls 133 | // s2sub( ) it won't be instanced by Excel. 134 | RTDServer rtd = RTDServer.GetInstance( ); 135 | if (rtd == null) { 136 | Logr.Log( String.Format( "UpdateRTD: no RTD server!") ); 137 | return; 138 | } 139 | string stopic = String.Format( "cron.{0}.{1}", qkey, subelem ); 140 | rtd.CacheUpdate( stopic, value ); 141 | } 142 | 143 | protected void OnTimerEvent( object o, ElapsedEventArgs e ) { 144 | m_Count++; 145 | ScheduleTimer( ); 146 | Logr.Log( String.Format( "OnTimerEvent count({0}) last({1}) next({2})", 147 | m_Count, m_LastEventTime, m_NextEventTime )); 148 | UpdateRTD( m_Key, "count", Convert.ToString( m_Count) ); 149 | UpdateRTD( m_Key, "last", m_LastEventTime ); 150 | UpdateRTD( m_Key, "next", m_NextEventTime ); 151 | } 152 | 153 | #endregion Pool thread 154 | } 155 | 156 | protected Dictionary m_CronMap = new Dictionary( ); 157 | 158 | #region Excel thread 159 | 160 | public static CronManager Instance( ) { 161 | // Unlikley that two threads will attempt to instance this singleton at the 162 | // same time, but we'll lock just in case. 163 | lock (s_InstanceLock) { 164 | if (s_Instance == null) { 165 | s_Instance = new CronManager( ); 166 | } 167 | return s_Instance; 168 | } 169 | } 170 | 171 | protected CronManager( ) { 172 | 173 | } 174 | 175 | public bool AddCron( string ckey, string cronex, DateTime? start, DateTime? end) { 176 | // no locking here as we won't touch any objects that are shared with another thread 177 | string[] cronflds = new string[6]; 178 | Logr.Log( String.Format( "AddCron: cronex({0}) start({1}) end({2})", cronex, start, end) ); 179 | try { 180 | if (m_CronMap.ContainsKey( ckey )) { 181 | // If there's already a timer with ckey it may be that an Excel users has triggered 182 | // another invocation of s2cron( ) by editting the s2cfg sheet, or with a sh-ctrl-alt-F9. 183 | // Either way, we need to remove the old timer, and create a new one, but only if the 184 | // new one is different. 185 | CronTimer oldTimer = m_CronMap[ckey]; 186 | if (oldTimer.Cronex == cronex && oldTimer.Start == start && oldTimer.End == end) { 187 | // no change, so we won't overwrite the entry for ckey 188 | return true; 189 | } 190 | m_CronMap.Remove( ckey); 191 | oldTimer.Close( ); 192 | } 193 | 194 | m_CronMap[ckey] = new CronTimer( ckey, cronex, start, end ); 195 | return true; 196 | } 197 | catch (Exception ex) { 198 | Logr.Log( String.Format( "AddCron: {0}", ex.Message ) ); 199 | return false; 200 | } 201 | } 202 | 203 | public void Clear( ) { 204 | // Called from RTDServer.ServerTerminate( ), which gets called when a sheet is shut down. 205 | // That means we have to stop the timers associated with that sheet. There is a bug lurking here, 206 | // because if a single Excel process has two workbooks open, and both create cron timers, and then 207 | // one is shut, both sets of timers will be cleared down. We'll leave that corner case for later... 208 | foreach (KeyValuePair entry in m_CronMap) { 209 | entry.Value.Close( ); // stop the timer 210 | } 211 | m_CronMap.Clear( ); 212 | } 213 | 214 | #endregion Excel thread 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /src/TWSCallback.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Net; 6 | using System.Text; 7 | using WebSocket4Net; 8 | using Newtonsoft.Json; 9 | using SuperSocket.ClientEngine; 10 | 11 | namespace SSAddin { 12 | 13 | // Apart from the ctor every method in TWSCallback that reads/writes object state locks 14 | // because we may have the worker background thread in touching object state as well as 15 | // a pool thread firing by a socket callback. 16 | class TWSCallback { 17 | public delegate void ClosedCB( string wskey ); 18 | 19 | protected WebSocket m_Client; 20 | protected ClosedCB m_ClosedCB; 21 | protected String m_AuthToken; 22 | protected String m_Key; 23 | protected String m_URL; 24 | protected String m_ProxyHost; 25 | protected String m_ProxyPort; 26 | protected String m_ProxyUser; 27 | protected String m_ProxyPassword; 28 | protected String m_SubID; 29 | protected Dictionary> m_Subscriptions = new Dictionary>( ); 30 | protected TiingoRealTimeMessageHandler m_RTMHandler; 31 | protected SortedSet m_PendingSubs = new SortedSet( ); 32 | // m_MktDataRecord field names must match up with https://api.tiingo.com/docs/iex/realtime 33 | // type: Q for quote, T for trade 34 | protected string[] m_MktDataRecord = { "type", "timestamp", "tickid", "ticker", "bidsz", "bid", "mid", "ask", "asksz", "ltrade", "ltradesz"}; 35 | protected int m_TickerIndex = 3; // index of "ticker" in m_MktDataRecord 36 | 37 | protected static DataCache s_Cache = DataCache.Instance( ); 38 | // Braces are special chars in C# format strings, so we need a double brace to indicate a literal single brace, 39 | // rather than a the start of a place holder eg {0} 40 | // This is the old s_SubscribeMessageFormat from before Rishi introduced subscriptionIDs 41 | // protected static String s_SubscribeMessageFormat = "{{\"eventName\":\"subscribe\",\"eventData\":{{\"authToken\": \"{0}\"}}}}"; 42 | protected static String s_SubscribeMessageFormat = "{{ \"eventName\":\"subscribe\",\"authorization\":\"{0}\",\"eventData\":{{ {1} }} }}"; 43 | // Format for composing {1} in s_SubscribeMessageFormat. 44 | protected static String s_EventDataFormat = "\"thresholdLevel\":0,\"tickers\":[{0}]"; 45 | protected static String s_EventDataSubIdFormat = "\"subscriptionId\":{0},\"thresholdLevel\":0,\"tickers\":[{1}]"; 46 | 47 | protected static JsonSerializerSettings s_JsonSettings = new JsonSerializerSettings( ) { 48 | Converters = { new JsonToDictionary( ) }, 49 | NullValueHandling = NullValueHandling.Ignore 50 | }; 51 | 52 | #region Worker thread 53 | 54 | public TWSCallback( Dictionary work, ClosedCB ccb ) { 55 | // No need to bother locking in the ctor. We are on the background 56 | // worker thread here, but we won't get methods fired on the pool 57 | // threads until this method exits. 58 | m_Key = work["key"]; 59 | m_URL = work["url"]; 60 | work.TryGetValue( "auth_token", out m_AuthToken); 61 | m_ClosedCB = ccb; 62 | try { 63 | m_Client = new WebSocket( m_URL); 64 | m_RTMHandler = new TiingoRealTimeMessageHandler(m_Client, MktDataTick, HeartBeat, SetSubID); 65 | m_Client.Opened += new EventHandler( Opened ); 66 | m_Client.Error += new EventHandler( Error ); 67 | m_Client.Closed += new EventHandler( Closed ); 68 | m_Client.MessageReceived += new EventHandler( MessageReceived ); 69 | m_Client.DataReceived += new EventHandler( DataReceived ); 70 | // Do we need to set up a proxy? 71 | if (work.TryGetValue("http_proxy_host", out m_ProxyHost)) { 72 | IPHostEntry he = Dns.GetHostEntry(m_ProxyHost); 73 | int port = 80; 74 | if (work.TryGetValue("http_proxy_host", out m_ProxyPort)) { 75 | if (!Int32.TryParse(m_ProxyPort, out port)) 76 | port = 80; 77 | } 78 | var proxy = new HttpConnectProxy( new IPEndPoint( he.AddressList[0], port)); 79 | // Do we need to supply authentication to the proxy? 80 | if (work.TryGetValue("http_proxy_user", out m_ProxyUser)) 81 | { 82 | work.TryGetValue("http_proxy_password", out m_ProxyPassword); 83 | // encode user:password as base64 and supply as 'Proxy-Authorization: Basic dXNlbWU6dGVzdA==' 84 | string upass = String.Format("{0}:{1}", m_ProxyUser, m_ProxyPassword); 85 | var plainTextBytes = System.Text.Encoding.UTF8.GetBytes(upass); 86 | string b64 = System.Convert.ToBase64String(plainTextBytes); 87 | proxy.Authorization = String.Format("Basic {0}", b64); 88 | } 89 | m_Client.Proxy = proxy; 90 | } 91 | Logr.Log(String.Format("TWSCallback.ctor: key({0}) url({1}) auth_token({2}) http_proxy_host({3}) http_proxy_port({4}) http_proxy_user({5}) http_proxy_password({6})", 92 | m_Key, m_URL, m_AuthToken, m_ProxyHost, m_ProxyPort, m_ProxyUser, m_ProxyPassword)); 93 | m_Client.Open( ); 94 | } 95 | catch (System.ArgumentException ae) { 96 | Logr.Log( String.Format( "TWSCallback.ctor: {0}", ae.Message ) ); 97 | } 98 | catch (System.FormatException fe) { 99 | Logr.Log( String.Format( "TWSCallback.ctor: format error fmt({0}) auth({1}) err({2})", s_SubscribeMessageFormat, m_AuthToken, fe.Message ) ); 100 | } 101 | } 102 | 103 | public void AddSubscriptions( List> subs) 104 | { 105 | // AddSubscriptions will be called from the background worker thread, but all the 106 | // object state it touches here could also be touched by the pool thread methods 107 | // below, so we lock on the socket object. 108 | lock (m_Client) { 109 | // First, put each subscription into the {ticker:[sub1,sub2] map 110 | // that tracks all existing ticker_sub subscriptions. 111 | // We're looking for eg twebsock.iex.appl_mid and we want to ignore 112 | // twebsock.iex.hbount. 113 | foreach (var sub in subs) { 114 | if ( sub.ContainsKey("cachekey") && sub.ContainsKey("ticker_field")) { 115 | string ckey = sub["cachekey"]; 116 | string tfld = sub["ticker_field"]; 117 | string[] tflds = tfld.Split('_'); 118 | if (ckey == m_Key && tflds.Length == 2) { 119 | string ticker = tflds[0]; 120 | string subelem = tflds[1]; 121 | SortedSet ss = null; 122 | if (!m_Subscriptions.ContainsKey( ticker )) { 123 | m_Subscriptions[ticker] = new SortedSet( ); 124 | m_PendingSubs.Add( ticker ); 125 | } 126 | ss = m_Subscriptions[ticker]; 127 | ss.Add( subelem ); 128 | } 129 | } 130 | } 131 | } 132 | // A worksheet invocation of s2sub could cause the background thread to dispatch here at 133 | // any time. Maybe because the user has edited a sheet to add a new s2sub( ). Any new 134 | // subs that have accumulated as a result should be dispatched as eearly as possible. 135 | DispatchSubscriptions( ); 136 | } 137 | 138 | #endregion Worker thread 139 | 140 | #region Worker or pool thread 141 | void DispatchSubscriptions( ) { 142 | // We could be called by either thread, so lock as we'll potentially change object state 143 | // and send stuff down the socker while another thread wants to do the same. 144 | lock (m_Client) { 145 | if (m_Client.State == WebSocketState.Open) { 146 | // The socket is open, so the Opened( ) callback below must have already fired. 147 | // An open socket isn't enough. We also need a subID, which we only have after 148 | // we've processed the response to the initial subscription. 149 | StringBuilder sb = new StringBuilder( ); 150 | int inx = 0; 151 | foreach (string sub in m_PendingSubs) { 152 | if (inx > 0) 153 | sb.Append(","); 154 | sb.Append( String.Format( "\"{0}\"", sub ) ); 155 | inx++; 156 | } 157 | string sublist = sb.ToString( ); 158 | string ed; 159 | if (m_SubID != null) { 160 | // We've got a subID, so only send a message if we've got tickers to add. 161 | if (sublist.Length == 0) 162 | return; 163 | ed = String.Format(s_EventDataSubIdFormat, m_SubID, sublist); 164 | } 165 | else { 166 | // We don't have a subID, so compose an initial message and send whether or not 167 | // we have a ticker list. 168 | ed = String.Format(s_EventDataFormat, sublist); 169 | } 170 | string submsg = String.Format( s_SubscribeMessageFormat, m_AuthToken, ed ); 171 | Logr.Log( String.Format( "TWSCallback.DispatchSubscriptions: subscribe message({0})", submsg ) ); 172 | m_Client.Send( submsg ); 173 | m_PendingSubs.Clear(); 174 | } 175 | } 176 | } 177 | #endregion Worker or pool thread 178 | 179 | #region Pool thread 180 | 181 | // All the pool thread methods are callbacks that will be fired on 182 | // web socket events on pool threads. 183 | 184 | protected void UpdateRTD( string key, string subelem, string value ) { 185 | // The RTD server doesn't necessarily exist. If no cell calls 186 | // s2sub( ) it won't be instanced by Excel. 187 | RTDServer rtd = RTDServer.GetInstance( ); 188 | if (rtd == null) 189 | return; 190 | string stopic = String.Format( "twebsock.{0}.{1}", key, subelem ); 191 | rtd.CacheUpdate( stopic, value ); 192 | } 193 | 194 | void DataReceived( object sender, WebSocket4Net.DataReceivedEventArgs e ) { 195 | Logr.Log( String.Format( "TWSCallback.DataReceived: {0}", e.Data ) ); 196 | } 197 | 198 | void MessageReceived( object sender, MessageReceivedEventArgs e ) { 199 | Logr.Log( String.Format( "TWSCallback.MessageReceived: {0}", e.Message ) ); 200 | var msg = JsonConvert.DeserializeObject>( e.Message, s_JsonSettings ); 201 | m_RTMHandler.MessageReceived(msg); // may cause callbacks to HeartBeat or MktDataTick below 202 | } 203 | 204 | public void HeartBeat(int hb) { 205 | UpdateRTD( m_Key, "hbcount", hb.ToString( )); 206 | } 207 | 208 | public void MktDataTick( IList tick) 209 | { 210 | if (tick.Count < m_MktDataRecord.Length) { 211 | Logr.Log(String.Format("TWSCallback.MktDataTick: fld count under {0}! {1}", 212 | m_MktDataRecord.Length, tick.ToString( ))); 213 | return; 214 | } 215 | string ticker = tick[m_TickerIndex].ToString(); 216 | lock (m_Client) { 217 | if (!m_Subscriptions.ContainsKey(ticker)) 218 | return; 219 | SortedSet fldset = m_Subscriptions[ticker]; 220 | for ( int inx = 0; inx < m_MktDataRecord.Length; inx++) { 221 | string fld = m_MktDataRecord[inx]; 222 | object val = tick[inx]; 223 | if ( fldset.Contains( fld) && val != null) { 224 | UpdateRTD( m_Key, String.Format( "{0}_{1}", ticker, fld), tick[inx].ToString( )); 225 | } 226 | } 227 | } 228 | // TODO: add TWSCache to s_Cache so that we only need an RTD sub to one field in a record, for 229 | // instance bid, and then the rest can be pulled from the cache... 230 | // s_Cache.UpdateWSCache( m_Key, updates ); 231 | } 232 | 233 | void Closed( object sender, EventArgs e ) { 234 | Logr.Log( String.Format( "TWSCallback.Closed: wskey({0})", m_Key ) ); 235 | if (m_ClosedCB != null) 236 | m_ClosedCB( String.Format( "twebsock.{0}", m_Key)); 237 | lock (m_Client) { 238 | // Socket has closed, and will need to be reopened. The reopen will trigger 239 | // another initial subscription, and then a new sub ID. 240 | m_SubID = null; 241 | } 242 | UpdateRTD( m_Key, "status", "closed" ); 243 | } 244 | 245 | void Error( object sender, SuperSocket.ClientEngine.ErrorEventArgs e ) { 246 | Logr.Log( String.Format( "TWSCallback.Error: wskey({0}) {1}", m_Key, e.Exception.Message ) ); 247 | UpdateRTD( m_Key, "status", "error" ); 248 | UpdateRTD( m_Key, "error", e.Exception.Message ); 249 | } 250 | 251 | void Opened( object sender, EventArgs e ) { 252 | DispatchSubscriptions( ); 253 | UpdateRTD( m_Key, "status", "open" ); 254 | } 255 | 256 | void SetSubID( string subID ) { 257 | // This method gets called on a pool thread by TiingoRealTimeMessageHandler 258 | lock (m_Client) { 259 | m_SubID = subID; 260 | } 261 | DispatchSubscriptions( ); 262 | } 263 | 264 | #endregion Pool thread 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /src/WorksheetFunctions.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Babbington Slade 2 | 3 | using System; 4 | using System.Diagnostics; 5 | using System.Collections.Generic; 6 | using System.Globalization; 7 | using System.IO; 8 | using System.Net; 9 | using ExcelDna.Integration; 10 | 11 | // Some notes on threading: there are multiple threads in play in this code, so locking is used fairly 12 | // freely in the code, and #region/#endregion markers are used to indicate which threads they're running on. 13 | // There are three kinds of threads... 14 | // 1. The Excel thread: the worksheet functions in this .cs run on the Excel thread. 15 | // 2. The background worker thread: this thread is launched in SSWebClient. It's top 16 | // level loop is in SSWebClient.BackgroundWork( ). SSWebClient.AddRequest( ) is used 17 | // by the Excel thread to pass work via a queue to this thread. This thread handles 18 | // HTTP GET style queries synchronously, so we avoid blocking the Excel thread. It 19 | // also initiates the websock and tiingo websock objects. 20 | // 3. .Net pool threads: .Net dispatches all kinds of events on pool threads: timers and 21 | // socket events. Consequently a lot of the cron code as well as websock and tiingo 22 | // websock callbacks run on pool threads. They dump results in caches that the Excel 23 | // thread accesses, so locking is needed. They also send results back to Excel via 24 | // the RTDServer; Excel is the RTD client. The RTDServer has methods that are invoked on 25 | // the Excel thread, so locking is necessary there too. 26 | // JOS 2016-05-25 27 | 28 | namespace SSAddin { 29 | public static class WorksheetFunctions { 30 | #region Fields 31 | static ConfigSheet s_ConfigSheet = new ConfigSheet( ); 32 | static DataCache s_Cache = DataCache.Instance( ); 33 | static SSWebClient s_WebClient = SSWebClient.Instance( ); 34 | static CronManager s_CronMgr = CronManager.Instance( ); 35 | static string s_Submitted = "OK"; 36 | static double s_SSAddinVersion = 0.42; 37 | #endregion 38 | 39 | #region Regular worksheet functions 40 | 41 | [ExcelFunction( Description = "Version info." )] 42 | public static object s2about( ) { 43 | return String.Format( "SSAddin {0} Excel {1}", s_SSAddinVersion, ExcelDnaUtil.ExcelVersion ); 44 | } 45 | 46 | [ExcelFunction( Description = "Launch quandl query.")] 47 | public static object s2quandl( 48 | [ExcelArgument( Name = "QueryKey", Description = "quandl query key in s2cfg!C" )] string qkey, 49 | [ExcelArgument( Name = "Trigger", Description = "dummy to trigger recalc" )] object trigger ) 50 | { 51 | String url = s_ConfigSheet.GetQueryURL( "quandl", qkey ); 52 | if ( url == null || url == "") { 53 | return ExcelMissing.Value; 54 | } 55 | var req = new Dictionary(){ {"type","quandl"}, {"key",qkey},{"url",url}}; 56 | s_ConfigSheet.GetProxyConfig("quandl", req); 57 | if (s_WebClient.AddRequest( req)) 58 | return s_Submitted; 59 | return ExcelError.ExcelErrorGettingData; 60 | } 61 | 62 | [ExcelFunction( Description = "Launch tiingo query." )] 63 | public static object s2tiingo( 64 | [ExcelArgument( Name = "QueryKey", Description = "tiingo query key in s2cfg!C" )] string qkey, 65 | [ExcelArgument( Name = "Trigger", Description = "dummy to trigger recalc" )] object trigger ) { 66 | String url = s_ConfigSheet.GetQueryURL( "tiingo", qkey ); 67 | if (url == null || url == "") { 68 | return ExcelMissing.Value; 69 | } 70 | // quandl puts the auth_token in the URL. tiingo is different, and puts it in the HTTP headers 71 | String auth_token = s_ConfigSheet.GetQueryConfig( "tiingo", "auth_token" ); 72 | if (auth_token == null || auth_token == "") { 73 | return ExcelMissing.Value; 74 | } 75 | var req = new Dictionary(){ {"type","tiingo"}, {"key",qkey},{"url",url},{"auth_token",auth_token}}; 76 | s_ConfigSheet.GetProxyConfig("tiingo", req); 77 | if (s_WebClient.AddRequest( req)) 78 | return s_Submitted; 79 | return ExcelError.ExcelErrorGettingData; 80 | } 81 | 82 | [ExcelFunction(Description = "Launch baremetrics query.")] 83 | public static object s2baremetrics( 84 | [ExcelArgument(Name = "QueryKey", Description = "baremetrics query key in s2cfg!C")] string qkey, 85 | [ExcelArgument(Name = "Trigger", Description = "dummy to trigger recalc")] object trigger) 86 | { 87 | String url = s_ConfigSheet.GetQueryURL("baremetrics", qkey); 88 | if (url == null || url == "") 89 | { 90 | return ExcelMissing.Value; 91 | } 92 | // baremetrics puts the auth_token in the HTTP headers 93 | String auth_token = s_ConfigSheet.GetQueryConfig("baremetrics", "auth_token"); 94 | if (auth_token == null || auth_token == "") 95 | { 96 | return ExcelMissing.Value; 97 | } 98 | var req = new Dictionary() { { "type", "baremetrics" }, { "key", qkey }, { "url", url }, { "auth_token", auth_token } }; 99 | s_ConfigSheet.GetProxyConfig("baremetrics", req); 100 | if (s_WebClient.AddRequest(req)) 101 | return s_Submitted; 102 | return ExcelError.ExcelErrorGettingData; 103 | } 104 | 105 | [ExcelFunction(Description = "Launch Google Analytics query.")] 106 | public static object s2ganalytics( 107 | [ExcelArgument(Name = "QueryKey", Description = "Google Analytics query key in s2cfg!C")] string qkey, 108 | [ExcelArgument(Name = "Trigger", Description = "dummy to trigger recalc")] object trigger) 109 | { 110 | Dictionary qterms = s_ConfigSheet.GetTerms( "ganalytics", "query", qkey ); 111 | if (qterms == null ) 112 | { 113 | return ExcelMissing.Value; 114 | } 115 | // Google Analytics needs a P12 key: auth_token should give us the path 116 | String auth_token = s_ConfigSheet.GetQueryConfig("ganalytics", "auth_token"); 117 | if (auth_token == null || auth_token == "") 118 | { 119 | return ExcelMissing.Value; 120 | } 121 | // Google Analytics needs a service account email address 122 | String service_account = s_ConfigSheet.GetQueryConfig("ganalytics", "id"); 123 | if (service_account == null || service_account == "") 124 | { 125 | return ExcelMissing.Value; 126 | } 127 | qterms["type"] = "ganalytics"; 128 | qterms["key"] = qkey; 129 | qterms["auth_token"] = auth_token; 130 | qterms["id"] = service_account; 131 | s_ConfigSheet.GetProxyConfig("ganalytics", qterms); 132 | if (s_WebClient.AddRequest(qterms)) 133 | return s_Submitted; 134 | return ExcelError.ExcelErrorGettingData; 135 | } 136 | 137 | [ExcelFunction( Description = "Schedule cron job." )] 138 | public static object s2cron( 139 | [ExcelArgument( Name = "CronKey", Description = "cron tab key in s2cfg!C" )] string ckey) 140 | { 141 | Tuple tup = s_ConfigSheet.GetCronTab( ckey ); 142 | if (tup == null) { 143 | return ExcelMissing.Value; 144 | } 145 | if (s_CronMgr.AddCron( ckey, tup.Item1, tup.Item2, tup.Item3)) 146 | return s_Submitted; 147 | return ExcelError.ExcelErrorValue; 148 | } 149 | 150 | [ExcelFunction( Description = "Connect to web socket." )] 151 | public static object s2websock( 152 | [ExcelArgument( Name = "SockKey", Description = "websock url key in s2cfg!C" )] string wskey ) { 153 | String url = s_ConfigSheet.GetWebSock( wskey ); 154 | if (url == null) { 155 | return ExcelMissing.Value; 156 | } 157 | if (s_WebClient.AddRequest( new Dictionary(){ {"type","websock"}, {"key",wskey},{"url",url}})) 158 | return s_Submitted; 159 | return ExcelError.ExcelErrorValue; 160 | } 161 | 162 | [ExcelFunction( Description = "Connect to tiingo web socket." )] 163 | public static object s2twebsock( 164 | [ExcelArgument( Name = "SockKey", Description = "twebsock url key in s2cfg!C" )] string wskey ) 165 | { 166 | Dictionary req = s_ConfigSheet.GetTiingoWebSock( wskey); 167 | if ( req == null) { 168 | return ExcelMissing.Value; 169 | } 170 | if (s_WebClient.AddRequest( req)) 171 | return s_Submitted; 172 | return ExcelError.ExcelErrorValue; 173 | } 174 | 175 | [ExcelFunction( Description = "Connect to transficc web socket." )] 176 | public static object s2transficc( 177 | [ExcelArgument( Name = "SockKey", Description = "transficc url key in s2cfg!C" )] string wskey ) { 178 | Dictionary req = s_ConfigSheet.GetTransficcWebSock( wskey ); 179 | if (req == null) { 180 | return ExcelMissing.Value; 181 | } 182 | if (s_WebClient.AddRequest( req )) 183 | return s_Submitted; 184 | return ExcelError.ExcelErrorValue; 185 | } 186 | 187 | [ExcelFunction( Description = "Pull data from S2 quandl cache.")] 188 | public static object s2qcache( 189 | [ExcelArgument(Name="QueryKey", Description="quandl query key in s2cfg!C")] string qkey, 190 | [ExcelArgument(Name="Trigger", Description="dummy to trigger recalc")] object trigger) 191 | { 192 | if ( !s_Cache.ContainsQuandlKey( qkey)) { 193 | return ExcelMissing.Value; 194 | } 195 | // Figure out our caller's posn in the sheet; that's the cell we'll pull from the cache. 196 | // If offsets are supplied use them to calc cell posn too. xoffset & yoffset will default 197 | // to 0 if not supplied in the sheet. 198 | ExcelReference caller = XlCall.Excel( XlCall.xlfCaller) as ExcelReference; 199 | int xoffset = s_ConfigSheet.GetQueryConfigAsInt("quandl", qkey, "xoffset"); 200 | int yoffset = s_ConfigSheet.GetQueryConfigAsInt("quandl", qkey, "yoffset"); 201 | string val = s_Cache.GetQuandlCell( qkey, caller.RowFirst-yoffset, caller.ColumnFirst-xoffset); 202 | if ( val == null ) { 203 | return s_ConfigSheet.GetDefaultCacheValue( "quandl", qkey ); 204 | } 205 | return val; 206 | } 207 | 208 | [ExcelFunction( Description = "Volatile: pull data from S2 quandl cache.", IsVolatile = true )] 209 | public static object s2vqcache( 210 | [ExcelArgument( Name = "QueryKey", Description = "quandl query key in s2cfg!C" )] string qkey) 211 | { 212 | return s2qcache( qkey, null ); 213 | } 214 | 215 | [ExcelFunction( Description = "Pull data from S2 tiingo cache." )] 216 | public static object s2tcache( 217 | [ExcelArgument( Name = "QueryKey", Description = "tiingo query key in s2cfg!C" )] string qkey, 218 | [ExcelArgument( Name = "Trigger", Description = "dummy to trigger recalc" )] object trigger ) 219 | { 220 | if (!s_Cache.ContainsTiingoKey( qkey )) { 221 | return ExcelMissing.Value; 222 | } 223 | // Figure out our caller's posn in the sheet; that's the cell we'll pull from the cache. 224 | // If offsets are supplied use them to calc cell posn too. xoffset & yoffset will default 225 | // to 0 if not supplied in the sheet. 226 | ExcelReference caller = XlCall.Excel( XlCall.xlfCaller ) as ExcelReference; 227 | int xoffset = s_ConfigSheet.GetQueryConfigAsInt("tiingo", qkey, "xoffset"); 228 | int yoffset = s_ConfigSheet.GetQueryConfigAsInt("tiingo", qkey, "yoffset"); 229 | string val = s_Cache.GetTiingoCell( qkey, caller.RowFirst - yoffset, caller.ColumnFirst - xoffset ); 230 | if (val == null) { 231 | return s_ConfigSheet.GetDefaultCacheValue( "tiingo", qkey ); 232 | } 233 | return val; 234 | } 235 | 236 | [ExcelFunction( Description = "Volatile: pull data from S2 tiingo cache.", IsVolatile = true )] 237 | public static object s2vtcache( 238 | [ExcelArgument( Name = "QueryKey", Description = "quandl query key in s2cfg!C" )] string qkey) 239 | { 240 | return s2tcache( qkey, null ); 241 | } 242 | 243 | [ExcelFunction(Description = "Pull data from S2 Google Analytics cache.")] 244 | public static object s2gacache( 245 | [ExcelArgument(Name = "QueryKey", Description = "ganalytics query key in s2cfg!C")] string qkey, 246 | [ExcelArgument(Name = "Trigger", Description = "dummy to trigger recalc")] object trigger) 247 | { 248 | if (!s_Cache.ContainsGAnalyticsKey(qkey)) 249 | { 250 | return ExcelMissing.Value; 251 | } 252 | // Figure out our caller's posn in the sheet; that's the cell we'll pull from the cache. 253 | // If offsets are supplied use them to calc cell posn too. xoffset & yoffset will default 254 | // to 0 if not supplied in the sheet. 255 | ExcelReference caller = XlCall.Excel(XlCall.xlfCaller) as ExcelReference; 256 | int xoffset = s_ConfigSheet.GetQueryConfigAsInt("ganalytics", qkey, "xoffset"); 257 | int yoffset = s_ConfigSheet.GetQueryConfigAsInt("ganalytics", qkey, "yoffset"); 258 | bool headers = Convert.ToBoolean( s_ConfigSheet.GetQueryConfigAsInt( "ganalytics", qkey, "headers" )); 259 | string val = s_Cache.GetGAnalyticsCell(qkey, caller.RowFirst - yoffset, caller.ColumnFirst - xoffset, headers); 260 | if (val == null) 261 | { 262 | return s_ConfigSheet.GetDefaultCacheValue( "ganalytics", qkey ); 263 | } 264 | return val; 265 | } 266 | 267 | [ExcelFunction(Description = "Volatile: pull data from S2 Google Analytics cache.", IsVolatile = true)] 268 | public static object s2vgacache( 269 | [ExcelArgument(Name = "QueryKey", Description = "ganalytics query key in s2cfg!C")] string qkey) 270 | { 271 | return s2gacache(qkey, null); 272 | } 273 | 274 | [ExcelFunction( Description = "Pull data from S2 baremetrics cache." )] 275 | public static object s2bcache( 276 | [ExcelArgument( Name = "QueryKey", Description = "baremetrics query key in s2cfg!C" )] string qkey, 277 | [ExcelArgument( Name = "Date", Description = "date key into result set. Use s2today, not Excel's TODAY" )] int xldate, 278 | [ExcelArgument( Name = "Field", Description = "field in the result set" )] string field, 279 | [ExcelArgument( Name = "Trigger", Description = "dummy to trigger recalc" )] object trigger ) { 280 | if (!s_Cache.ContainsBareKey( qkey )) { 281 | return ExcelMissing.Value; 282 | } 283 | string val = s_Cache.GetBareField( qkey, s_ConfigSheet.ExcelDateNumberToString( xldate), field); 284 | if (val == null) { 285 | return s_ConfigSheet.GetDefaultCacheValue( "baremetrics", qkey ); 286 | } 287 | return val; 288 | } 289 | 290 | [ExcelFunction( Description = "Volatile: pull data from S2 baremetrics cache.", IsVolatile = true )] 291 | public static object s2vbcache( 292 | [ExcelArgument( Name = "QueryKey", Description = "baremetrics query key in s2cfg!C" )] string qkey, 293 | [ExcelArgument( Name = "Date", Description = "date key into result set. Use s2today, not Excel's TODAY" )] int xldate, 294 | [ExcelArgument( Name = "Field", Description = "field in the result set" )] string field ) { 295 | return s2bcache( qkey, xldate, field, null ); 296 | } 297 | 298 | [ExcelFunction( Description = "Non volatile alternate to Excel's TODAY.")] 299 | public static object s2today( 300 | [ExcelArgument( Name = "Offset", Description = "days +/- from today" )] int offset, 301 | [ExcelArgument( Name = "Trigger", Description = "dummy to trigger recalc" )] object trigger ) { 302 | DateTime dt = DateTime.Now.AddDays( Convert.ToDouble( offset ) ); 303 | return dt.ToString( "yyyy-MM-dd" ); 304 | } 305 | 306 | [ExcelFunction( Description = "Pull data from S2 web socket cache." )] 307 | public static object s2wscache( 308 | [ExcelArgument( Name = "QueryKey", Description = "websock query key in s2cfg!C" )] string wkey, 309 | [ExcelArgument( Name = "CellKey", Description = "m2_6_0 for col 3, row 7 on first sheet" )] string ckey, 310 | [ExcelArgument( Name = "Trigger", Description = "dummy to trigger recalc" )] object trigger ) { 311 | // Figure out our caller's posn in the sheet; that's the cell we'll pull from the cache. 312 | // If offsets are supplied use them to calc cell posn too. xoffset & yoffset will default 313 | // to 0 if not supplied in the sheet. 314 | string val = s_Cache.GetWSCell( wkey, ckey); 315 | if (val == null) { 316 | return s_ConfigSheet.GetDefaultCacheValue( "websock", wkey ); 317 | } 318 | return val; 319 | } 320 | 321 | [ExcelFunction( Description = "Volatile: pull data from S2 web socket cache.", IsVolatile = true)] 322 | public static object s2vwscache( 323 | [ExcelArgument( Name = "QueryKey", Description = "websock query key in s2cfg!C" )] string wkey, 324 | [ExcelArgument( Name = "CellKey", Description = "m2_6_0 for col 3, row 7 on first sheet" )] string ckey) { 325 | return s2wscache( wkey, ckey, null ); 326 | } 327 | 328 | #endregion 329 | 330 | #region RTD functions 331 | [ExcelFunction( Description = "RTD: Subscribe to properties of S2 cache." )] 332 | public static object s2sub( 333 | [ExcelArgument( Name = "SubCache", Description = "[quandl|tiingo|cron|websock|twebsock]" )] string subcache, 334 | [ExcelArgument(Name="CacheKey", Description="Row key from s2cfg")] string ckey, 335 | [ExcelArgument(Name="Property", Description="[status|count|next|last|mX_Y_Z|ticker_field]")] string prop) 336 | { 337 | string[] arrey = { subcache, ckey, prop}; 338 | string stopic = String.Join( ".", arrey); 339 | Logr.Log( String.Format( "s2sub: {0}", stopic)); 340 | // Send a message to the worker thread about this subscription. It may need to fwd 341 | // to another object eg TWSCallback for subscription management. 342 | var sdict = new Dictionary() { { "type", "s2sub" }, { "key", stopic }, 343 | { "subcache", subcache }, { "ticker_field", prop}, { "cachekey", ckey} }; 344 | s_WebClient.AddRequest( sdict); 345 | // Make the RTD call to Excel or SpreadServe's internal RTD API to let it know 346 | // about the new subscription. 347 | return XlCall.RTD( "SSAddin.RTDServer", null, stopic); 348 | } 349 | #endregion 350 | } 351 | } 352 | -------------------------------------------------------------------------------- /src/ConfigSheet.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.Linq; 5 | using System.Text; 6 | using ExcelDna.Integration; 7 | 8 | // https://www.quandl.com/api/v1/datasets/WIKI/AAPL.csv 9 | // https://www.quandl.com/api/v1/datasets/WIKI/AAPL.json?trim_start=1985-05-01&trim_end=1997-07-01&sort_order=asc&column=4&collapse=quarterly&transformation=rdiff 10 | 11 | 12 | namespace SSAddin { 13 | 14 | // Wrapper for all the config info in the s2cfg sheet, if it exists 15 | 16 | class ConfigSheet { 17 | 18 | protected static String s_QuandlBaseURL = "https://www.quandl.com/api/v1/datasets"; 19 | protected static String s_TiingoBaseURL = "https://api.tiingo.com/tiingo"; 20 | protected static String s_BareBaseURL = "https://api.baremetrics.com/v1/metrics"; 21 | protected static String s_BareSandBaseURL = "https://api-sandbox.baremetrics.com/v1/metrics"; 22 | protected static Dictionary> s_QuandlQueryFieldConverters = new Dictionary>( ); 23 | protected static Dictionary> s_TiingoQueryFieldConverters = new Dictionary>( ); 24 | protected static Dictionary> s_BareQueryFieldConverters = new Dictionary>(); 25 | protected static string[] s_ProxyKeys = { "http_proxy_host", "http_proxy_port", "http_proxy_user", "http_proxy_password" }; 26 | 27 | protected Dictionary m_ConfigCache = new Dictionary(); 28 | protected Dictionary m_DefaultValueCache = new Dictionary( ); 29 | 30 | public ConfigSheet( ) { 31 | // Quandl and Tiingo funcs expect Excel dates. In the GUI Excel recognises the date, and converts to a number 32 | s_QuandlQueryFieldConverters["trim_start"] = ExcelDateNumberToString; 33 | s_QuandlQueryFieldConverters["trim_end"] = ExcelDateNumberToString; 34 | s_TiingoQueryFieldConverters["startDate"] = ExcelDateNumberToString; 35 | s_TiingoQueryFieldConverters["endDate"] = ExcelDateNumberToString; 36 | // For baremetrics queries date params can be supplied by s2today, which arrives in here as 37 | // a string, not a number like an Excel date. Or they can be an Excel date. 38 | s_BareQueryFieldConverters["start_date"] = ExcelDateNumberToString; 39 | s_BareQueryFieldConverters["end_date"] = ExcelDateNumberToString; 40 | } 41 | 42 | public object GetCell( int row, int col ) { 43 | ExcelReference xlref = new ExcelReference( row, row, col, col, "s2cfg" ); 44 | return xlref.GetValue( ); 45 | } 46 | 47 | public String GetCellAsString( int row, int col ) { 48 | object val = GetCell( row, col); 49 | if ( val == ExcelEmpty.Value) 50 | return ""; 51 | return val.ToString( ); 52 | } 53 | 54 | public string ExcelDateNumberToString( object dn ) { 55 | double d = 0.0; 56 | string sdt = dn.ToString( ); 57 | if ( !Double.TryParse( sdt, out d)) 58 | return sdt; 59 | DateTime dt = DateTime.FromOADate( d); 60 | // "u" format is 2008-06-15 21:15:07Z 61 | sdt = dt.ToString( "u" ); 62 | // Throw away the time and just keep the date... 63 | return sdt.Substring( 0, 10 ); 64 | } 65 | 66 | public String BuildQuandlQuery( Dictionary qterms ) { 67 | if ( !qterms.ContainsKey( "dataset")) 68 | return ""; 69 | StringBuilder sb = new StringBuilder( s_QuandlBaseURL); 70 | sb.Append( String.Format( "/{0}.csv", qterms["dataset"])); 71 | qterms.Remove( "dataset"); 72 | string prefix = "?"; 73 | string auth_token = GetQueryConfig( "quandl", "auth_token" ); 74 | if (auth_token != "") { 75 | sb.Append( String.Format( "{0}{1}={2}", prefix, "auth_token", auth_token ) ); 76 | prefix = "&"; 77 | } 78 | string val; 79 | foreach ( KeyValuePair item in qterms) { 80 | if (s_QuandlQueryFieldConverters.ContainsKey( item.Key )) { 81 | Func converter = s_QuandlQueryFieldConverters[item.Key]; 82 | val = converter( item.Value ); 83 | } 84 | else { 85 | val = item.Value.ToString( ); 86 | } 87 | sb.Append( String.Format( "{0}{1}={2}", prefix, item.Key, val)); 88 | prefix = "&"; 89 | } 90 | return sb.ToString( ); 91 | } 92 | 93 | public String BuildTiingoQuery( Dictionary qterms ) { 94 | // Must specify a ticker symbol and a root (daily|funds) 95 | if (!qterms.ContainsKey( "ticker" ) || !qterms.ContainsKey("root")) 96 | return ""; 97 | // Now deal with the minimal essential we need to build a valid tiingo query 98 | StringBuilder sb = new StringBuilder( s_TiingoBaseURL ); 99 | sb.Append( String.Format( "/{0}/{1}", qterms["root"], qterms["ticker"] ) ); 100 | qterms.Remove( "ticker" ); 101 | qterms.Remove( "root" ); 102 | if (qterms.ContainsKey( "leaf" )) { 103 | sb.Append( String.Format( "/{0}", qterms["leaf"] ) ); 104 | qterms.Remove( "leaf" ); 105 | } 106 | // Are there any more values in the Dict? Maybe startDate and endDate... 107 | if (qterms.Count > 0) { 108 | string prefix = "?"; 109 | string val; 110 | foreach (KeyValuePair item in qterms) { 111 | if (s_TiingoQueryFieldConverters.ContainsKey( item.Key )) { 112 | Func converter = s_TiingoQueryFieldConverters[item.Key]; 113 | val = converter( item.Value ); 114 | } 115 | else { 116 | val = item.Value.ToString( ); 117 | } 118 | sb.Append( String.Format( "{0}{1}={2}", prefix, item.Key, val ) ); 119 | prefix = "&"; 120 | } 121 | } 122 | return sb.ToString( ); 123 | } 124 | 125 | public String BuildBareQuery(Dictionary qterms) 126 | { 127 | // Must specify a query type 128 | if (!qterms.ContainsKey("qtype")) 129 | return ""; 130 | // Now deal with the minimal essential we need to build a valid baremetrics query 131 | bool sandbox = false; 132 | if (qterms.ContainsKey("sandbox")) { 133 | try { 134 | sandbox = Convert.ToBoolean(qterms["sandbox"]); 135 | } 136 | catch ( Exception ex) { 137 | Logr.Log(String.Format("BuildBareQuery: bad sandbox value {0}\n{1}", qterms["sandbox"].ToString( ), ex)); 138 | } 139 | qterms.Remove( "sandbox" ); 140 | } 141 | StringBuilder sb = new StringBuilder(sandbox ? s_BareSandBaseURL : s_BareBaseURL); 142 | String qtype = qterms["qtype"].ToString( ); 143 | if (qtype == "summary") { 144 | // null op for summary as we append the start and end dates below 145 | } 146 | else if (qtype == "metric") { 147 | if ( qterms.ContainsKey( "metric")) { 148 | sb.Append( String.Format( "/{0}", qterms["metric"])); 149 | qterms.Remove( "metric" ); 150 | } 151 | else { 152 | Logr.Log(String.Format("BuildBareQuery: qtype==metric, but no metric specified eg metric=mrr")); 153 | return ""; 154 | } 155 | } 156 | else if (qtype == "plans" || qtype == "customers") { 157 | if (qterms.ContainsKey( "metric" )) { 158 | sb.Append( String.Format( "/{0}/{1}", qterms["metric"], qtype ) ); 159 | qterms.Remove( "metric" ); 160 | } 161 | else { 162 | Logr.Log( String.Format( "BuildBareQuery: qtype=={0}, but no metric specified eg metric=mrr", qtype ) ); 163 | return ""; 164 | } 165 | } 166 | else { 167 | Logr.Log( String.Format( "BuildBareQuery: bad qtype=={0}", qtype ) ); 168 | return ""; 169 | } 170 | qterms.Remove("qtype"); 171 | // Are there any more values in the Dict? Maybe start_date and end_date... 172 | if (qterms.Count > 0) 173 | { 174 | string prefix = "?"; 175 | string val; 176 | foreach (KeyValuePair item in qterms) 177 | { 178 | if (s_BareQueryFieldConverters.ContainsKey(item.Key)) 179 | { 180 | Func converter = s_BareQueryFieldConverters[item.Key]; 181 | val = converter(item.Value); 182 | } 183 | else 184 | { 185 | val = item.Value.ToString(); 186 | } 187 | sb.Append(String.Format("{0}{1}={2}", prefix, item.Key, val)); 188 | prefix = "&"; 189 | } 190 | } 191 | return sb.ToString(); 192 | } 193 | 194 | public int FindRow( string c0, string c1, string c2, int startRow = 0) 195 | { 196 | // We're looking for a row that has c0 in the first cell, c1 in the second, 197 | // and then c2 in the third. 198 | int row = startRow; 199 | string a, b, c; 200 | do { // keep going as long as the first field in a row isn't empty 201 | a = GetCellAsString( row, 0 ); 202 | b = GetCellAsString( row, 1 ); 203 | c = GetCellAsString( row, 2 ); 204 | if ( a == c0 && b == c1 && c == c2) 205 | return row; 206 | row++; 207 | } while (a != null && a != ""); 208 | return -1; 209 | } 210 | 211 | public int FindRow(string c0, string c1, string c2, string c3) 212 | { 213 | // We're looking for a row that has c0 in the first cell, c1 in the second, 214 | // c2 in the third and c3 in the fourth 215 | string d; 216 | int row = FindRow(c0, c1, c2, 0); 217 | while ( row != -1) { 218 | d = GetCellAsString(row, 3); 219 | if (d == c3) 220 | return row; 221 | row = FindRow(c0, c1, c2, row + 1); 222 | } 223 | return row; 224 | } 225 | 226 | public String GetQueryURL( String qtype, String qkey) { 227 | Dictionary qterms = GetTerms( qtype, "query", qkey ); 228 | if ( qtype == "quandl") 229 | return BuildQuandlQuery( qterms ); 230 | if (qtype == "baremetrics") 231 | return BuildBareQuery(qterms); 232 | return BuildTiingoQuery( qterms ); 233 | } 234 | 235 | public Dictionary GetTerms( String qtype, String col2, String qkey ) { 236 | // We're looking for a row that has 'quandl', 'tiingo', 'baremetrics' or ganalytics in the first cell, 237 | // query in the second, and then qkey in the third. 238 | int row = FindRow( qtype, col2, qkey ); 239 | if (row == -1) { 240 | Logr.Log( String.Format( "GetTerms: couldn't find {0} {1} {2}", qtype, col2, qkey ) ); 241 | return null; 242 | } 243 | return GetKeyValPairs( row); 244 | } 245 | 246 | protected Dictionary GetKeyValPairs( int row) 247 | { 248 | int col = 3; 249 | string name; 250 | string val; 251 | Dictionary qterms = new Dictionary(); 252 | do { 253 | name = GetCellAsString(row, col); 254 | val = GetCellAsString(row, col + 1); 255 | if (name != null && name != "") 256 | qterms.Add(name, val); 257 | col += 2; 258 | } while (name != null && name != ""); 259 | return qterms; 260 | } 261 | 262 | public String GetQueryConfig(String qtype, String ckey) 263 | { 264 | // Get config for all queries of type qtype. For instance, 265 | // the auth_key for all quandl queries. 266 | string xkey = String.Format("{0}.{1}", qtype, ckey); 267 | if (m_ConfigCache.ContainsKey(xkey)) 268 | { 269 | return m_ConfigCache[xkey]; 270 | } 271 | // Look for config in .Net .config file too. Could be auth_token or http_proxy, 272 | // because we don't want to expose it in the sheet, and we don't want to repeat 273 | // in info in every sheet. 274 | string token = ConfigurationManager.AppSettings.Get( xkey ); 275 | if (token != null && token != "") { 276 | m_ConfigCache[xkey] = token; 277 | Logr.Log( String.Format( "GetQueryConfig: using ssaddin.xll.config for {0}:{1}", xkey, token ) ); 278 | return token; 279 | } 280 | // We're looking for a row that has qtype [quandl|tiingo] in the first cell, config in the second, 281 | // and then ckey in the third. 282 | int row = FindRow( qtype, "config", ckey ); 283 | if (row == -1) { 284 | Logr.Log( String.Format( "GetQueryConfig: couldn't find {0}.{1}", qtype, ckey ) ); 285 | // Caching "" as a value will ensure we get this logged once only 286 | m_ConfigCache[xkey] = ""; 287 | return ""; 288 | } 289 | string val = GetCellAsString( row, 3 ); 290 | Logr.Log( String.Format( "GetQueryConfig: returning row {0} col 3 value:{1}", row, val ) ); 291 | m_ConfigCache[xkey] = val; 292 | return val; 293 | } 294 | 295 | public object GetDefaultCacheValue( String qtype, String qkey ) { 296 | // Get default cache value for all qkey query type qtype. For instance, 297 | // qtype=="baremetrics", qkey=="summaryQuery1" 298 | string xkey = String.Format( "{0}.{1}.default_cache_value", qtype, qkey ); 299 | if (m_DefaultValueCache.ContainsKey( xkey )) { 300 | return m_DefaultValueCache[xkey]; 301 | } 302 | // We won't look for default cache values config in .Net .config file too. This is very much 303 | // a per sheet piece of config, not per host like auth keys. We're looking for a row that has 304 | // qtype [quandl|tiingo|baremetrics] in the first cell, config in the second, qkey in the third 305 | // and default_cache_value in the fourth. 306 | int row = FindRow( qtype, "config", qkey, "default_cache_value" ); 307 | if (row == -1) { 308 | Logr.Log( String.Format( "GetDefaultCacheValue: couldn't find {0}", xkey ) ); 309 | // Caching "" as a value will ensure we get this logged once only 310 | m_DefaultValueCache[xkey] = ExcelError.ExcelErrorNA; 311 | return ExcelError.ExcelErrorNA; 312 | } 313 | string val = GetCellAsString( row, 4 ); 314 | Logr.Log( String.Format( "GetDefaultCacheValue: returning row {0} col 4 value:{1}", row, val ) ); 315 | m_DefaultValueCache[xkey] = val; 316 | return val; 317 | } 318 | 319 | public int GetQueryConfigAsInt(String qtype, String qkey, String ckey) 320 | { 321 | // Get config for a specific query qkey, rather than all queries of type qtype. 322 | // For example, get xoffset (ckey) for a ganalytics (qtype) query called 323 | // metrics1 (qkey). 324 | int rv = 0; 325 | string xkey = String.Format("{0}.{1}.{2}", qtype, qkey, ckey); 326 | if (m_ConfigCache.ContainsKey(xkey)) { 327 | if (Int32.TryParse(m_ConfigCache[xkey], out rv)) 328 | return rv; 329 | } 330 | // Look for config in .Net .config file too. Could be auth_token or http_proxy, 331 | // because we don't want to expose it in the sheet, and we don't want to repeat 332 | // in info in every sheet. 333 | string token = ConfigurationManager.AppSettings.Get(xkey); 334 | if (token != null && token != "") { 335 | m_ConfigCache[xkey] = token; 336 | if (Int32.TryParse(token, out rv)) 337 | Logr.Log(String.Format("GetQueryConfigAsInt: using ssaddin.xll.config for {0}:{1}", xkey, token)); 338 | else 339 | Logr.Log(String.Format("GetQueryConfigAsInt: INT ERROR ssaddin.xll.config for {0}:{1}", xkey, token)); 340 | return rv; 341 | } 342 | // We're looking for a row that has qtype [quandl|tiingo] in the first cell, config in the second, 343 | // and then qkey in the third. 344 | int row = FindRow(qtype, "config", qkey, ckey); 345 | if (row != -1) { 346 | string val = GetCellAsString(row, 4); 347 | m_ConfigCache[xkey] = val; 348 | if (Int32.TryParse(val, out rv)) { 349 | Logr.Log(String.Format("GetQueryConfigAsInt: returning row {0} col 4 value:{1}", row, val)); 350 | return rv; 351 | } 352 | Logr.Log(String.Format("GetQueryConfigAsInt: INT ERROR row {0} col 3 value:{1}", row, val)); 353 | // Caching "" as a value will ensure we get this logged once only 354 | m_ConfigCache[xkey] = ""; 355 | } 356 | return rv; 357 | } 358 | 359 | public Tuple GetCronTab( String ctabkey ) { 360 | // We're looking for a row that has 'cron' in the first cell, tab in the second, 361 | // and then ctabkey in the third. 362 | int row = FindRow( "cron", "tab", ctabkey ); 363 | if (row == -1) { 364 | Logr.Log( String.Format( "GetCronTab: couldn't find {0}", ctabkey ) ); 365 | return null; 366 | } 367 | // Now we've found the right row we expect to find six columns to make up a 368 | // crontab entry in D, E, F, G, H, I, J, and then two more columns for start 369 | // & end in K & L 370 | string[] flds = new string[6]; 371 | int col = 0; 372 | for ( ; col < 6; col++) 373 | flds[col] = GetCellAsString( row, col + 3 ); 374 | string cronex = String.Join( " ", flds ); 375 | double dstart, dend; 376 | DateTime? start = null; // return nulls if K & L cols are empty 377 | DateTime? end = null; 378 | // new DateTime( start.Year, start.Month, start.Day, 23, 59, 59 ); 379 | // If the start and end cells on the cron tab entry on the s2cfg page are time of 380 | // day eg 09:30:00 and not full date times then they yield DateTime doubles that 381 | // are < 1.0 as they encode no date/day info. But the Interval arithmetic for the 382 | // next event in CronManager uses DateTime.Now as a baseline, and that includes 383 | // date/day info. So we must baseline off the date/day for today too. 384 | DateTime sod = new DateTime( DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0 ); 385 | double dsod = sod.ToOADate( ); 386 | string sstart = GetCellAsString( row, 3 + col++ ); 387 | string send = GetCellAsString( row, 3 + col++); 388 | if (Double.TryParse( sstart, out dstart )) { 389 | if ( dstart < 1.0) 390 | dstart += dsod; 391 | start = DateTime.FromOADate( dstart ); 392 | } 393 | if (Double.TryParse( send, out dend )) { 394 | if (dend < 1.0) 395 | dend += dsod; 396 | end = DateTime.FromOADate( dend ); 397 | } 398 | return new Tuple( cronex, start, end); 399 | } 400 | 401 | public String GetWebSock( String wskey ) { 402 | // We're looking for a row that has 'websock' in the first cell, url in the second, 403 | // and then wskey in the third. 404 | int row = FindRow( "websock", "url", wskey ); 405 | if (row == -1) { 406 | Logr.Log( String.Format( "GetWebSock: couldn't find {0}", wskey ) ); 407 | return null; 408 | } 409 | // Now we've found the right row we expect to find three columns to make up a 410 | // URL: host, port, path in D, E & F 411 | string host = GetCellAsString( row, 3); 412 | string port = GetCellAsString( row, 4); 413 | string path = GetCellAsString( row, 5); 414 | if (host == null || port == null || path == null) { 415 | Logr.Log( String.Format( "GetWebSock: bad host, port or path wskey({0})", wskey)); 416 | return null; 417 | } 418 | string url = String.Format( "ws://{0}:{1}/{2}", host, port, path ); 419 | return url; 420 | } 421 | 422 | public Dictionary GetTiingoWebSock( String wskey ) { 423 | // We're looking for a row that has 'twebsock' in the first cell, tiingo in the second, 424 | // and then wskey in the third. 425 | int row = FindRow( "twebsock", "tiingo", wskey ); 426 | if (row == -1) { 427 | Logr.Log( String.Format( "GetTiingoWebSock: couldn't find {0}", wskey ) ); 428 | return null; 429 | } 430 | // Now we've found the right row we expect to find the url in col D 431 | // and the auth_token on a config row. The auth_token is mandatory 432 | // for a tiingo websock. 433 | string url = GetCellAsString( row, 3 ); 434 | string auth_token = GetQueryConfig( "tiingo", "auth_token" ); 435 | if (url == null || auth_token == null) { 436 | Logr.Log( String.Format( "GetTiingoWebSock: bad url or auth_token wskey({0})", wskey ) ); 437 | return null; 438 | } 439 | var req = new Dictionary( ){{"type","twebsock"},{"key",wskey},{"url",url},{"auth_token",auth_token}}; 440 | // Now let's deal with optional elements of a tiingo request: proxy config may be supplied if we have to 441 | // connect via a proxy. 442 | GetProxyConfig("tiingo", req); 443 | return req; 444 | } 445 | 446 | public Dictionary GetTransficcWebSock( String wskey ) { 447 | // We're looking for a row that has 'trasficc' in the first cell, marketdata in the second, 448 | // and then wskey in the third. 449 | int row = FindRow( "transficc", "marketdata", wskey ); 450 | if (row == -1) { 451 | Logr.Log( String.Format( "GetTransficcWebSock: couldn't find {0}", wskey ) ); 452 | return null; 453 | } 454 | // Now we've found the right row we expect to find the url in col D 455 | string url = GetCellAsString( row, 3 ); 456 | if (url == null) { 457 | Logr.Log( String.Format( "GetTransficcWebSock: bad url wskey({0})", wskey ) ); 458 | return null; 459 | } 460 | var req = new Dictionary( ) { { "type", "transficc" }, { "key", wskey }, { "url", url }, { "subtype", "marketdata" } }; 461 | return req; 462 | } 463 | 464 | public void GetProxyConfig(string ctype, Dictionary req) 465 | { 466 | foreach (string key in s_ProxyKeys) 467 | { 468 | string val = GetQueryConfig(ctype, key); 469 | if (val != "") 470 | req.Add(key, val); 471 | } 472 | } 473 | } 474 | } 475 | -------------------------------------------------------------------------------- /src/SSWebClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading; 6 | using System.IO; 7 | using System.Net; 8 | using System.Diagnostics; 9 | using NCrontab; 10 | using Newtonsoft.Json; 11 | using Newtonsoft.Json.Linq; 12 | 13 | namespace SSAddin { 14 | #region Deserialization classes 15 | // tiingo historical data format 16 | class SSTiingoHistPrice { 17 | public string date { get; set; } 18 | public float open { get; set; } 19 | public float close { get; set; } 20 | public float high { get; set; } 21 | public float low { get; set; } 22 | public float volume { get; set; } 23 | public float adjOpen { get; set; } 24 | public float adjClose { get; set; } 25 | public float adjHigh { get; set; } 26 | public float adjLow { get; set; } 27 | public float adjVolume { get; set; } 28 | } 29 | // baremetrics metrics query responses 30 | // https://developers.baremetrics.com/reference#available-metrics 31 | class BareMetricsSummaryArrayElement { 32 | public string human_date { get; set; } 33 | public string date { get; set; } 34 | public int active_customers { get; set; } 35 | public int active_subscriptions { get; set; } 36 | public int add_on_mrr { get; set; } 37 | public int arpu { get; set; } 38 | public int arr { get; set; } 39 | public int cancellations { get; set; } 40 | public int coupons { get; set; } 41 | public int downgrades { get; set; } 42 | public int failed_charges { get; set; } 43 | public int fees { get; set; } 44 | public int ltv { get; set; } 45 | public int mrr { get; set; } 46 | public int net_revenue { get; set; } 47 | public int new_customers { get; set; } 48 | public int other_revenue { get; set; } 49 | public int reactivated_customers { get; set; } 50 | public int refunds { get; set; } 51 | public int revenue_churn { get; set; } 52 | public int trial_conversions { get; set; } 53 | public int upgrades { get; set; } 54 | public int user_churn { get; set; } 55 | } 56 | 57 | class BareMetricsSummary { 58 | public List metrics { get; set; } 59 | } 60 | 61 | #endregion 62 | 63 | class SSWebClient { 64 | protected static DataCache s_Cache = DataCache.Instance( ); 65 | protected static char[] csvDelimiterChars = { ',' }; 66 | protected static SSWebClient s_Instance; 67 | protected static object s_InstanceLock = new object( ); 68 | 69 | protected static JsonSerializerSettings s_JsonSettings = new JsonSerializerSettings( ) { 70 | NullValueHandling = NullValueHandling.Ignore, 71 | MissingMemberHandling = MissingMemberHandling.Ignore 72 | }; 73 | 74 | protected Queue> m_InputQueue; 75 | protected HashSet m_InFlight; 76 | protected Dictionary m_WSCallbacks; // Emulates SpreadServe JS webpage 77 | protected TWSCallback m_TWSCallback; // Tiingo API client 78 | protected TFWSCallback m_TFWSCallback; // transficc client 79 | 80 | // If s2sub( ) creates subscription requests before s2twebsock( ) or 81 | // s2tranficc( ) has been called queue them up in these caches 82 | protected List> m_PendingTiingoSubs; 83 | protected List> m_PendingTransficcSubs; 84 | 85 | protected Thread m_WorkerThread; // for executing the web query 86 | protected ManualResetEvent m_Event; // control worker thread sleep 87 | protected String m_TempDir; 88 | protected int m_QuandlCount; // total number of quandl queries 89 | protected int m_TiingoCount; // total number of tiingo queries 90 | protected int m_BareCount; // total number of baremetrics queries 91 | protected int m_GACount; // total number of ganalytics queries 92 | 93 | #region Excel thread methods 94 | 95 | protected SSWebClient( ) 96 | { 97 | m_TempDir = System.IO.Path.GetTempPath( ); 98 | m_InputQueue = new Queue>( ); 99 | m_InFlight = new HashSet( ); 100 | m_WSCallbacks = new Dictionary( ); 101 | m_Event = new ManualResetEvent( false); 102 | m_PendingTiingoSubs = new List>( ); 103 | m_PendingTransficcSubs = new List>( ); 104 | m_WorkerThread = new Thread( BackgroundWork ); 105 | m_WorkerThread.Start( ); 106 | m_QuandlCount = 0; 107 | m_TiingoCount = 0; 108 | m_BareCount = 0; 109 | m_GACount = 0; 110 | 111 | // Push out an RTD update for the overall quandl query count. This will mean 112 | // that trigger parms driven by quandl.all.count don't have #N/A as input for 113 | // long, which will enable eg ycb_pub_quandl.xls qlPiecewiseYieldCurve to 114 | // calc almost immediately. JOS 2015-07-29 115 | UpdateRTD( "quandl", "all", "count", String.Format( "{0}", m_QuandlCount++)); 116 | UpdateRTD( "tiingo", "all", "count", String.Format( "{0}", m_TiingoCount++ ) ); 117 | UpdateRTD( "baremetrics", "all", "count", String.Format("{0}", m_BareCount++)); 118 | } 119 | 120 | public static SSWebClient Instance( ) { 121 | // Unlikley that two threads will attempt to instance this singleton at the 122 | // same time, but we'll lock just in case. 123 | lock (s_InstanceLock) { 124 | if (s_Instance == null) { 125 | s_Instance = new SSWebClient( ); 126 | } 127 | return s_Instance; 128 | } 129 | } 130 | 131 | 132 | public bool AddRequest( Dictionary req) { 133 | // Every request *must* have type, key, url. There may be other optionals like 134 | // auth_token, https_proxy_host... 135 | string type = req["type"]; 136 | string key = req["key"]; 137 | string fkey = String.Format( "{0}.{1}", req["type"], req["key"]); 138 | bool isWebQuery = (type == "quandl" || type == "tiingo" || type == "baremetrics" || type == "ganalytics"); 139 | // Is this job pending or in progress? 140 | lock (m_InFlight) { 141 | if (m_InFlight.Contains( fkey )) { // Queued or running... 142 | if ( isWebQuery) 143 | Logr.Log( String.Format( "~A AddRequest: {0} is already inflight", fkey ) ); 144 | return false; // so bail 145 | } 146 | // Nested locking - look out! We're on the Excel thread here as we're invoked by 147 | // worksheet functions. The background worker thread does use this lock too, but 148 | // not at the same time as m_InFlight, so we should be OK. 149 | lock (m_InputQueue) { 150 | // Running on the main Excel thread here. Q the work, and 151 | // signal the background thread to wake up... 152 | if ( isWebQuery) 153 | Logr.Log( String.Format( "~A AddRequest adding {0} {1}", type, key ) ); 154 | m_InputQueue.Enqueue( req); 155 | } 156 | // NB some fkeys are only ever added to m_InFlight, and are never removed. For 157 | // instance the s2sub notifications. We only need an s2sub notification to go to 158 | // the background thread once to create websock subscriptions. And if it's an 159 | // hbcount for instance, there's no work on the background thread, so we'll 160 | // let it through once, then block subsequent notifies. We may want to revisit 161 | // this logic in future if we need to resubscribe to websock topics. But for 162 | // the time being let's work around by starting and stopping the sheet. 163 | // JOS 2016-05-26 164 | m_InFlight.Add( fkey); 165 | m_Event.Set( ); // signal worker thread to wake 166 | } 167 | return true; 168 | } 169 | 170 | #endregion 171 | 172 | #region Pool thread methods 173 | 174 | protected void WSCallbackClosed( string wskey) { 175 | lock (m_InFlight) { 176 | // removing the key of the websock allows another incoming request 177 | // with the same key 178 | m_InFlight.Remove( wskey ); 179 | m_WSCallbacks.Remove( wskey ); 180 | } 181 | } 182 | 183 | protected void TWSCallbackClosed( string wskey ) { 184 | lock (m_InFlight) { 185 | // releasing the existing tiingo websock callback handler enables the 186 | // BackgroundWork method to create another one 187 | m_InFlight.Remove( wskey ); 188 | m_TWSCallback = null; 189 | } 190 | } 191 | 192 | protected void TFWSCallbackClosed( string wskey ) { 193 | lock (m_InFlight) { 194 | // releasing the existing tiingo websock callback handler enables the 195 | // BackgroundWork method to create another one 196 | m_InFlight.Remove( wskey ); 197 | m_TFWSCallback = null; 198 | } 199 | } 200 | 201 | #endregion 202 | 203 | #region Worker thread methods 204 | 205 | protected Dictionary GetWork( ) { 206 | // Put this one liner in its own method to wrap the locking. We can't 207 | // hold the lock while we're looping in BackgroundWork( ) as that 208 | // will prevent the Excel thread adding new requests. 209 | lock ( m_InputQueue) { 210 | if (m_InputQueue.Count > 0) 211 | return m_InputQueue.Dequeue( ); 212 | return null; 213 | } 214 | } 215 | 216 | public void BackgroundWork( ) { 217 | // We're running on the background thread. Loop until we're told to exit... 218 | Logr.Log( String.Format( "~A BackgroundWork thread started")); 219 | // Main loop for worker thread. It will briefly hold the m_InFlight lock when 220 | // it removes entries, and also the m_InputQueue lock in GetWork( ) as it 221 | // removes entries. DoQuandlQuery( ) will grab the cache lock when it's 222 | // adding cache entries. Obviously, no lock should be held permanently! JOS 2015-04-31 223 | while (true) { 224 | // Wait for a signal from the other thread to say there's some work. 225 | m_Event.WaitOne( ); 226 | Dictionary work = GetWork( ); 227 | while ( work != null) { 228 | if (work["type"] == "stop") { 229 | // exit req from excel thread 230 | Logr.Log( String.Format( "~A BackgroundWork thread exiting" ) ); 231 | return; 232 | } 233 | string fkey = String.Format("{0}.{1}", work["type"], work["key"]); 234 | Logr.Log(String.Format("~A BackgroundWork new request flight_key({0})", fkey)); 235 | 236 | if (work["type"] == "quandl") { 237 | // run query synchronously here on background worker thread 238 | bool ok = DoQuandlQuery( work); 239 | // query done, so remove key from inflight, which will permit 240 | // the query to be resubmitted 241 | lock (m_InFlight) { 242 | m_InFlight.Remove( fkey ); 243 | } 244 | } 245 | else if (work["type"] == "tiingo") { 246 | // run query synchronously here on background worker thread 247 | bool ok = DoTiingoQuery( work ); 248 | // query done, so remove key from inflight, which will permit 249 | // the query to be resubmitted 250 | lock (m_InFlight) { 251 | m_InFlight.Remove( fkey ); 252 | } 253 | } 254 | else if (work["type"] == "baremetrics") 255 | { 256 | // run query synchronously here on background worker thread 257 | bool ok = DoBareQuery(work); 258 | // query done, so remove key from inflight, which will permit 259 | // the query to be resubmitted 260 | lock (m_InFlight) 261 | { 262 | m_InFlight.Remove(fkey); 263 | } 264 | } 265 | else if (work["type"] == "ganalytics") 266 | { 267 | // run query synchronously here on background worker thread 268 | bool ok = DoGAnalyticsQuery(work); 269 | // query done, so remove key from inflight, which will permit 270 | // the query to be resubmitted 271 | lock (m_InFlight) 272 | { 273 | m_InFlight.Remove(fkey); 274 | } 275 | } 276 | else if (work["type"] == "websock") { 277 | WSCallback wscb = new WSCallback( work, this.WSCallbackClosed ); 278 | // We don't want to remove the inflight key here as there will be 279 | // async callbacks to WSCallback on pool threads when updates 280 | // arrive on the SS websock. So we leave the key in place to 281 | // prevent AddRequest, which is on the Excel thread, creating 282 | // a request for a new WSCallback. 283 | lock (m_InFlight) { 284 | m_WSCallbacks.Add( fkey, wscb ); 285 | } 286 | } 287 | else if (work["type"] == "twebsock") { 288 | // We don't want to remove the inflight key here as there will be 289 | // async callbacks to TWSCallback on pool threads when updates 290 | // arrive on the tiingo websock. So we leave the key in place to 291 | // prevent AddRequest, which is on the Excel thread, creating 292 | // a request for a new TWSCallback. 293 | lock (m_InFlight) { 294 | if (m_TWSCallback == null) { 295 | m_TWSCallback = new TWSCallback( work, this.TWSCallbackClosed ); 296 | if (m_PendingTiingoSubs.Count > 0) { 297 | m_TWSCallback.AddSubscriptions(m_PendingTiingoSubs); 298 | m_PendingTiingoSubs.Clear(); 299 | } 300 | } 301 | } 302 | } 303 | else if (work["type"] == "transficc") { 304 | // We don't want to remove the inflight key here as there will be 305 | // async callbacks to TFWSCallback on pool threads when updates 306 | // arrive on the tiingo websock. So we leave the key in place to 307 | // prevent AddRequest, which is on the Excel thread, creating 308 | // a request for a new TFWSCallback. 309 | lock (m_InFlight) { 310 | if (m_TFWSCallback == null) { 311 | m_TFWSCallback = new TFWSCallback( work, this.TFWSCallbackClosed ); 312 | if (m_PendingTransficcSubs.Count > 0) { 313 | m_TFWSCallback.AddSubscriptions( m_PendingTransficcSubs ); 314 | m_PendingTransficcSubs.Clear( ); 315 | } 316 | } 317 | } 318 | } 319 | else if (work["type"] == "s2sub") { 320 | if (work["subcache"] == "twebsock") { 321 | // New subscription to a tiingo websock. If the TWSCallback 322 | // doesn't exist yet cache it, but if it does pass it through 323 | lock (m_InFlight) { 324 | m_PendingTiingoSubs.Add(work); 325 | if (m_TWSCallback != null) { 326 | m_TWSCallback.AddSubscriptions(m_PendingTiingoSubs); 327 | m_PendingTiingoSubs.Clear(); 328 | } 329 | } 330 | } 331 | else if (work["subcache"] == "transficc") { 332 | // New subscription to a transficc websock. If the TFWSCallback 333 | // doesn't exist yet cache it, but if it does pass it through 334 | lock (m_InFlight) { 335 | m_PendingTransficcSubs.Add( work ); 336 | if (m_TFWSCallback != null) { 337 | m_TFWSCallback.AddSubscriptions( m_PendingTransficcSubs ); 338 | m_PendingTransficcSubs.Clear( ); 339 | } 340 | } 341 | } 342 | } 343 | work = GetWork( ); 344 | } 345 | // We've exhausted the queued work, so reset the event so that we wait in the 346 | // WaitOne( ) invocation above until another thread signals that there's some 347 | // more work. 348 | m_Event.Reset( ); 349 | } 350 | } 351 | 352 | protected void UpdateRTD( string subcache, string qkey, string subelem, string value ) { 353 | // The RTD server doesn't necessarily exist. If no cell calls 354 | // s2sub( ) it won't be instanced by Excel. 355 | RTDServer rtd = RTDServer.GetInstance( ); 356 | if ( rtd == null) 357 | return; 358 | string stopic = String.Format( "{0}.{1}.{2}", subcache, qkey, subelem ); 359 | rtd.CacheUpdate( stopic, value ); 360 | } 361 | 362 | protected void ConfigureProxy(Dictionary work, WebClient wc) 363 | { 364 | // If the dictionary has proxy config, then set it up... 365 | if (!work.ContainsKey("http_proxy_host")) 366 | return; 367 | 368 | int port = 80; 369 | string host = work["http_proxy_host"]; 370 | if (work.ContainsKey("http_proxy_port")) 371 | { 372 | if (!Int32.TryParse(work["http_proxy_port"], out port)) 373 | port = 80; 374 | } 375 | 376 | WebProxy proxy = new WebProxy( String.Format("{0}:{1}", host, port), true); 377 | string user = "", pass = ""; 378 | if (work.ContainsKey("http_proxy_user") && work.ContainsKey("http_proxy_password")) { 379 | user = work["http_proxy_user"]; 380 | pass = work["http_proxy_password"]; 381 | proxy.Credentials = new NetworkCredential( user, pass); 382 | } 383 | wc.Proxy = proxy; 384 | Logr.Log(String.Format("ConfigureProxy host({0}) port({1}) user({2}) pass({3})", host, port, user, pass)); 385 | } 386 | 387 | protected bool DoQuandlQuery( Dictionary work) 388 | { 389 | string qkey = work["key"]; 390 | string url = work["url"]; 391 | string line = ""; 392 | string lineCount = "0"; 393 | try { 394 | // Set up the web client to HTTP GET 395 | var client = new WebClient( ); 396 | ConfigureProxy(work, client); 397 | Stream data = client.OpenRead( url); 398 | var reader = new StreamReader( data); 399 | // Local file to dump result 400 | int pid = Process.GetCurrentProcess( ).Id; 401 | string csvfname = String.Format( "{0}\\{1}_{2}.csv", m_TempDir, qkey, pid ); 402 | Logr.Log( String.Format( "running quandl qkey({0}) {1} persisted at {2}", qkey, url, csvfname)); 403 | var csvf = new StreamWriter( csvfname ); 404 | UpdateRTD( "quandl", qkey, "status", "starting" ); 405 | // Clear any previous result from the cache so we don't append repeated data 406 | s_Cache.ClearQuandl( qkey ); 407 | while ( reader.Peek( ) >= 0) { 408 | // For each CSV line returned by quandl, dump to localFS, add to in mem cache, and 409 | // send a line count update to any RTD subscriber 410 | line = reader.ReadLine( ); 411 | csvf.WriteLine( line ); 412 | lineCount = String.Format( "{0}", s_Cache.AddQuandlLine( qkey, line.Split( csvDelimiterChars))); 413 | UpdateRTD( "quandl", qkey, "count", lineCount ); 414 | } 415 | csvf.Close( ); 416 | data.Close(); 417 | reader.Close(); 418 | UpdateRTD( "quandl", qkey, "status", "complete" ); 419 | UpdateRTD( "quandl", "all", "count", String.Format( "{0}", m_QuandlCount++ ) ); 420 | Logr.Log( String.Format( "quandl qkey({0}) complete count({1})", qkey, lineCount)); 421 | return true; 422 | } 423 | catch( System.IO.IOException ex) { 424 | Logr.Log( String.Format( "quandl qkey({0}) url({1}) {2}", qkey, url, ex) ); 425 | } 426 | catch (System.Net.WebException ex) { 427 | Logr.Log( String.Format( "quandl qkey({0}) url({1}) {2}", qkey, url, ex ) ); 428 | } 429 | return false; 430 | } 431 | 432 | protected bool DoTiingoQuery( Dictionary work) { 433 | string qkey = work["key"]; 434 | string url = work["url"]; 435 | string auth_token = work["auth_token"]; 436 | string line = ""; 437 | string lineCount = "0"; 438 | try { 439 | // Set up the web client to HTTP GET 440 | var client = new WebClient( ); 441 | ConfigureProxy(work, client); 442 | client.Headers.Set( "Content-Type", "application/json" ); 443 | client.Headers.Set( "Authorization", String.Format("Token {0}", auth_token )); 444 | Stream data = client.OpenRead( url ); 445 | var reader = new StreamReader( data ); 446 | // Local file to dump result 447 | int pid = Process.GetCurrentProcess( ).Id; 448 | string jsnfname = String.Format( "{0}\\{1}_{2}.jsn", m_TempDir, qkey, pid ); 449 | Logr.Log( String.Format( "running tiingo qkey({0}) {1} persisted at {2}", qkey, url, jsnfname ) ); 450 | var jsnf = new StreamWriter( jsnfname ); 451 | UpdateRTD( "tiingo", qkey, "status", "starting" ); 452 | // Clear any previous result from the cache so we don't append repeated data 453 | s_Cache.ClearTiingo( qkey ); 454 | StringBuilder sb = new StringBuilder( ); 455 | while (reader.Peek( ) >= 0) { 456 | // For each json line returned by tiingo, dump to localFS, add to in mem cache 457 | line = reader.ReadLine( ); 458 | jsnf.WriteLine( line ); 459 | sb.AppendLine( line ); 460 | } 461 | jsnf.Close( ); 462 | data.Close( ); 463 | reader.Close( ); 464 | UpdateRTD( "tiingo", qkey, "status", "complete" ); 465 | UpdateRTD( "tiingo", "all", "count", String.Format( "{0}", m_TiingoCount++ ) ); 466 | Logr.Log( String.Format( "tiingo qkey({0}) complete count({1})", qkey, lineCount ) ); 467 | List updates = JsonConvert.DeserializeObject>( sb.ToString( ), s_JsonSettings); 468 | s_Cache.UpdateTHPCache( qkey, updates ); 469 | UpdateRTD( "tiingo", qkey, "count", String.Format( "{0}", updates.Count) ); 470 | return true; 471 | } 472 | catch (System.IO.IOException ex) { 473 | Logr.Log( String.Format( "tiingo qkey({0}) url({1}) {2}", qkey, url, ex ) ); 474 | } 475 | catch (System.Net.WebException ex) { 476 | Logr.Log( String.Format( "tiingo qkey({0}) url({1}) {2}", qkey, url, ex ) ); 477 | } 478 | return false; 479 | } 480 | 481 | protected bool DoBareQuery(Dictionary work) 482 | { 483 | string qkey = work["key"]; 484 | string url = work["url"]; 485 | string auth_token = work["auth_token"]; 486 | string line = ""; 487 | string lineCount = "0"; 488 | try 489 | { 490 | // Set up the web client to HTTP GET 491 | var client = new WebClient(); 492 | ConfigureProxy(work, client); 493 | client.Headers.Set("Content-Type", "application/json"); 494 | client.Headers.Set("Authorization", String.Format("Bearer {0}", auth_token)); 495 | Stream data = client.OpenRead(url); 496 | var reader = new StreamReader(data); 497 | // Local file to dump result 498 | int pid = Process.GetCurrentProcess().Id; 499 | string jsnfname = String.Format("{0}\\{1}_{2}.jsn", m_TempDir, qkey, pid); 500 | Logr.Log(String.Format("running baremetric qkey({0}) {1} persisted at {2}", qkey, url, jsnfname)); 501 | var jsnf = new StreamWriter(jsnfname); 502 | UpdateRTD("baremetrics", qkey, "status", "starting"); 503 | // Clear any previous result from the cache so we don't append repeated data 504 | StringBuilder sb = new StringBuilder(); 505 | while (reader.Peek() >= 0) 506 | { 507 | // For each json line returned by baremetrics, dump to localFS, add to in mem cache 508 | line = reader.ReadLine(); 509 | jsnf.WriteLine(line); 510 | sb.AppendLine(line); 511 | } 512 | jsnf.Close(); 513 | data.Close(); 514 | reader.Close(); 515 | UpdateRTD("baremetrics", qkey, "status", "complete"); 516 | UpdateRTD("baremetrics", "all", "count", String.Format("{0}", m_BareCount++)); 517 | Logr.Log(String.Format("baremetrics qkey({0}) complete count({1})", qkey, lineCount)); 518 | dynamic updates = JsonConvert.DeserializeObject( sb.ToString( ) ); 519 | s_Cache.UpdateBareCache(qkey, updates); 520 | UpdateRTD("baremetrics", qkey, "count", String.Format("{0}", updates.metrics.Count)); 521 | return true; 522 | } 523 | catch (System.IO.IOException ex) 524 | { 525 | Logr.Log(String.Format("baremetrics qkey({0}) url({1}) {2}", qkey, url, ex)); 526 | } 527 | catch (System.Net.WebException ex) 528 | { 529 | Logr.Log(String.Format("baremetrics qkey({0}) url({1}) {2}", qkey, url, ex)); 530 | } 531 | return false; 532 | } 533 | 534 | protected bool DoGAnalyticsQuery(Dictionary work) 535 | { 536 | string qkey = work["key"]; 537 | string metrics = work["metrics"]; 538 | string dimensions = work["dimensions"]; 539 | string start_date = work["start_date"]; 540 | string end_date = work["end_date"]; 541 | string lineCount = "0"; 542 | try 543 | { 544 | int pid = Process.GetCurrentProcess().Id; 545 | string csvfname = String.Format("{0}\\{1}_{2}.csv", m_TempDir, qkey, pid); 546 | var csvf = new StreamWriter(csvfname); 547 | Logr.Log(String.Format("running ganalytics qkey({0}) dimensions({1}) metrics({2}) start_date({3}) end_date({4}) {5}", 548 | qkey, dimensions, metrics, start_date, end_date, csvfname)); 549 | UpdateRTD("ganalytics", qkey, "status", "starting"); 550 | GoogleAnalyticsAPI gapi = GoogleAnalyticsAPI.Instance(work["auth_token"], work["id"]); 551 | AnalyticDataPoint adp = gapi.GetAnalyticsData(dimensions, metrics, start_date, end_date); 552 | foreach (IList row in adp.Rows) 553 | { 554 | csvf.WriteLine( String.Join( ",", row)); 555 | } 556 | csvf.Close(); 557 | UpdateRTD("ganalytics", qkey, "status", "complete"); 558 | UpdateRTD("ganalytics", "all", "count", String.Format("{0}", m_GACount++)); 559 | Logr.Log(String.Format("ganalytics qkey({0}) complete count({1})", qkey, lineCount)); 560 | s_Cache.UpdateGACache(qkey, adp); 561 | UpdateRTD("ganalytics", qkey, "count", String.Format("{0}", adp.Rows.Count)); 562 | return true; 563 | } 564 | catch (System.IO.IOException ex) 565 | { 566 | Logr.Log(String.Format("ganalytics qkey({0}) {1}", qkey, ex)); 567 | } 568 | catch (System.Net.WebException ex) 569 | { 570 | Logr.Log(String.Format("ganalytics qkey({0}) {1}", qkey, ex)); 571 | } 572 | return false; 573 | } 574 | 575 | #endregion 576 | } 577 | } 578 | --------------------------------------------------------------------------------