I have the below SQL Script which is run from Excel VBA using ADO against a SQL Server 2000 database.
The issue I have is that although there is only one SELECT statement in the script, I sometimes receive three recordsets back when executing the .Open method. I say sometimes as on occassion and on other ‘parallel’ databases I only receive one recordset back.
I know all about the .NextRecordset() method etc. but I am trying to understand why I get three recordsets back some times and other times I only receive one. I’m shortly going to run a SQL trace to see if that throws up any ideas but as usual, any help or advice would be greatly appreciated.
SET NOCOUNT ON
DECLARE @RunDate VARCHAR(8)
SET @RunDate = CONVERT(VARCHAR(8), DATEADD(d, -1 * 1, GETDATE()), 112)
IF OBJECT_ID('tempdb..#ActiveOrders') IS NOT NULL DROP TABLE #ActiveOrders
IF OBJECT_ID('tempdb..#ApplicableOrders') IS NOT NULL DROP TABLE #ApplicableOrders
IF OBJECT_ID('tempdb..#FilterOut') IS NOT NULL DROP TABLE #FilterOut
/*Temp table created as it has a self-join in the below query */
CREATE TABLE #ActiveOrders(
order_id VARCHAR(30)
, instrument_id VARCHAR(30)
, side CHAR(1)
)
CREATE INDEX idx_ActiveOrders_orderId ON #ActiveOrders(order_id)
/*Build dataset of all orders which have had activity on the run date or are in an Open status. Ignoring Program Trades.*/
INSERT INTO #ActiveOrders
SELECT o1.order_id COLLATE Latin1_General_CI_AS
, o1.instrument_id
, o1.side
FROM orders o1
INNER JOIN desk d1 ON d1.desk_id = o1.investment_desk
INNER JOIN (SELECT o0.order_id
FROM orders o0
WHERE ((LEFT(o0.added_datetime, 8) = @RunDate
OR LEFT(o0.approved_datetime, 8) = @RunDate)
OR (LEFT(o0.added_datetime, 8) <= @RunDate
AND o0.summary_status IN (1, 2, 3, 5, 8, 9))) /*Approved, Assigned, Acknowledged, Working, Partial, WorkingPartial*/
UNION
(SELECT r0.order_id
FROM releases r0
WHERE LEFT(r0.added_datetime, 8) = @RunDate)
UNION
(SELECT e0.order_id
FROM executions e0
WHERE LEFT(e0.execution_datetime, 8) = @RunDate
OR LEFT(e0.allocated_datetime, 8) = @RunDate)
) t1 ON o1.order_id = t1.order_id
WHERE d1.location_id = 'LDEQ'
AND o1.summary_status <> 4
AND o1.list_id IS NULL /*Ignore program trades*/
/*This is now the actual dataset we are interested in.
This is everything which could be a contender for aggregation.*/
CREATE TABLE #ApplicableOrders(
order_id VARCHAR(30)
, instrument_id VARCHAR(30)
, side CHAR(1)
, approved_datetime DATETIME
, acknowledged_datetime DATETIME
, last_allocation_datetime DATETIME
, latest_status INT
, merged_orders VARCHAR(500)
, dealer VARCHAR(100)
, manager VARCHAR(100)
, limit_price FLOAT
, original_qty FLOAT
, executed_qty FLOAT
, trader_instruction TEXT
, dealer_note TEXT
)
CREATE INDEX idx_ApplicableOrders_orderId ON #ApplicableOrders(order_id)
CREATE INDEX idx_ApplicableOrders_lastAllocation ON #ApplicableOrders(last_allocation_datetime)
CREATE INDEX idx_ApplicableOrders_approved ON #ApplicableOrders(approved_datetime)
/*All orders from #ActiveOrders where there are two or more orders which are for the same instrument and in the same direction.*/
INSERT INTO #ApplicableOrders
SELECT o.order_id
, o.instrument_id
, o.side
, dbo.mglz_datetime(o.approved_datetime)
, dbo.mglz_datetime(o.ack_datetime)
, MAX(dbo.mglz_datetime(e.allocated_datetime)) "Last Allocation DateTime"
, o.summary_status
, o.merged_orders
, o.ack_id
, o.approver_id
, o.limit_price
, o.original_qty
, o.executed_qty_at
, CONVERT(VARCHAR(900), o.trader_instruction)
, CONVERT(VARCHAR(900), o.dealer_note)
FROM orders o
INNER JOIN #ActiveOrders t ON o.order_id = t.order_id COLLATE Latin1_General_CI_AS
INNER JOIN #ActiveOrders s ON s.order_id <> o.order_id COLLATE Latin1_General_CI_AS
AND s.instrument_id = o.instrument_id COLLATE Latin1_General_CI_AS
AND s.side = o.side COLLATE Latin1_General_CI_AS
LEFT JOIN executions e ON e.order_id = o.order_id
GROUP BY o.order_id
, o.instrument_id
, o.side
, o.approved_datetime
, o.ack_datetime
, o.summary_status
, o.merged_orders
, o.ack_id
, o.approver_id
, o.limit_price
, o.original_qty
, o.executed_qty_at
, CONVERT(VARCHAR(900), o.trader_instruction)
, CONVERT(VARCHAR(900), o.dealer_note)
/*Filter out any orders where Order2.Approved_Date > Order1.Last_Release_Date AND Order1.Is_Complete
Order1 is defined as the order which was approved first.*/
SELECT t1.*
INTO #FilterOut
FROM #ApplicableOrders t1
WHERE EXISTS (SELECT 1
FROM
(SELECT order2.order_id
FROM (SELECT b.order_id
, b.instrument_id
, b.side
, b.approved_datetime
, b.last_allocation_datetime
, b.latest_status
, b.executed_qty
, b.original_qty
FROM #ApplicableOrders b
WHERE b.approved_datetime = (SELECT MIN(b1.approved_datetime) FirstApproval
FROM #ApplicableOrders b1
WHERE b1.instrument_id = b.instrument_id
AND b1.side = b.side)
) order1
INNER JOIN
(SELECT c.order_id
, c.instrument_id
, c.side
, c.approved_datetime
FROM #ApplicableOrders c
WHERE c.approved_datetime > (SELECT MIN(c1.approved_datetime) FirstApproval
FROM #ApplicableOrders c1
WHERE c1.instrument_id = c.instrument_id
AND c1.side = c.side)
) order2
ON order1.instrument_id = order2.instrument_id
AND order1.side = order2.side
AND order2.approved_datetime > order1.last_allocation_datetime
AND (order1.latest_status = 6 OR order1.executed_qty = order1.original_qty)) filter1
WHERE t1.order_id = filter1.order_id)
/*Filter out any orders where Order2.Acknowledged_Date > Order1.Last_Allocation_Date.*/
INSERT INTO #FilterOut
SELECT t1.*
FROM #ApplicableOrders t1
WHERE EXISTS (SELECT 1
FROM
(SELECT order2.order_id
FROM (SELECT b.order_id
, b.instrument_id
, b.side
, b.approved_datetime
, b.last_allocation_datetime
FROM #ApplicableOrders b
WHERE b.approved_datetime = (SELECT MIN(b1.approved_datetime) FirstApproval
FROM #ApplicableOrders b1
WHERE b1.instrument_id = b.instrument_id
AND b1.side = b.side)
) order1
INNER JOIN
(SELECT c.order_id
, c.instrument_id
, c.side
, c.approved_datetime
, c.acknowledged_datetime
FROM #ApplicableOrders c
WHERE c.approved_datetime > (SELECT MIN(c1.approved_datetime) FirstApproval
FROM #ApplicableOrders c1
WHERE c1.instrument_id = c.instrument_id
AND c1.side = c.side)
) order2
ON order1.instrument_id = order2.instrument_id
AND order1.side = order2.side
AND order2.acknowledged_datetime > order1.last_allocation_datetime) filter2
WHERE t1.order_id = filter2.order_id)
AND NOT EXISTS (SELECT 1
FROM #FilterOut a1
WHERE a1.order_id = t1.order_id)
/*Filter any 'single' orders. I.e. all 'matching' orders have been excluded so the instrument/direction combination is not applicable for Aggregation.*/
INSERT INTO #FilterOut
SELECT t1.*
FROM #ApplicableOrders t1 INNER JOIN (SELECT DISTINCT t.instrument_id
, t.side
FROM #ApplicableOrders t
INNER JOIN #FilterOut a ON t.instrument_id = a.instrument_id
AND t.side = a.side
GROUP BY t.instrument_id
, t.side
HAVING COUNT(t.instrument_id) > 1) t2 ON t1.instrument_id = t2.instrument_id
AND t1.side = t2.side
WHERE NOT EXISTS (SELECT 1
FROM #FilterOut a1
WHERE a1.order_id = t1.order_id)
/*Final Report*/
/*A list of all orders where aggregation could have possibly occurred but didn't.*/
SELECT t1.order_id "Order ID"
, i.name "Name"
, t1.side "B/S"
, userDealer.short_name "Dlr"
, userManager.short_name "FM"
, t1.limit_price "Limit"
, t1.approved_datetime "Order Approved"
, t1.acknowledged_datetime "Order Acknowledged"
, t1.last_allocation_datetime "Last Execution"
, t1.merged_orders "Merged Orders"
, m.description "Status"
, t1.dealer_note "Dealer Note"
, t1.trader_instruction "Trader Instruction"
FROM #ApplicableOrders t1
INNER JOIN instrument i ON t1.instrument_id = i.instrument_id COLLATE Latin1_General_CI_AS
INNER JOIN mnemonics m ON t1.latest_status = m.value AND m.attribute = 'order_summary_status'
LEFT JOIN users userDealer ON userDealer.user_id = t1.dealer COLLATE Latin1_General_CI_AS
LEFT JOIN users userManager ON userManager.user_id = t1.manager COLLATE Latin1_General_CI_AS
WHERE NOT EXISTS (SELECT 1
FROM #FilterOut t2
WHERE t1.order_id = t2.order_id)
ORDER BY t1.name
, t1.side
, t1.approved_datetime
IF OBJECT_ID('tempdb..#ActiveOrders') IS NOT NULL DROP TABLE #ActiveOrders
IF OBJECT_ID('tempdb..#ApplicableOrders') IS NOT NULL DROP TABLE #ApplicableOrders
IF OBJECT_ID('tempdb..#FilterOut') IS NOT NULL DROP TABLE #FilterOut
I finally found the issue and thought I would report back. The problem was with NULL values in two of the Aggregation Functions in the script. The reason this only occurred occasionally is because the data was constantly changing.
Although SQL Server handles the issue ‘silently’ it does produce warnings which ADO sees as recordsets although in my case (might be every case) these recordsets are closed and so impossible to actually see what they contain or what produced them.
Instead of re-writing the SQL script with ISNULL() logic, I have opted simply to include the
statement at the top of the script. This prevents the warnings from being reported and thus ADO does not produce these additional, unwanted recordsets.
Disappointed I didn’t spot this earlier but at least I’ve learnt something new. 🙂