Question

my stored procedure works fine on its own but my python script fails to fully execute the stored procedure with my downloaded files. the purpose of the python script is to download files using ftp and store the files locally.It first compares the remote location and the local location to find new files and then download the new files to the local location. And then executes the stored procedure on each new file.

python script:

import os
import ftplib
import pyodbc

connection to sql server*
conn = pyodbc.connect(‘DRIVER={SQL Server};SERVER=localhost;DATABASE=Development;UID=myid;PWD=mypassword’)
cursor = conn.cursor()

ftp = ftplib.FTP("myftpaddress.com")
ftp.login("loginname", "password")
print 'ftp on'

#directory listing 
rfiles = ftp.nlst()
print 'remote listing'

#save local directory listing to files
lfiles = os.listdir(r"D:\Raw_Data\myFiles")
print 'local listing'

#compare and find files in rfiles but not in lfiles
nfiles = set(rfiles) - set(lfiles)
nfiles = list(nfiles)
print 'compared listings'

#loop through the new files
#download the new files and open each file and run stored proc
#close files and disconnect to sql server
for n in nfiles:
   local_filename = os.path.join(r"D:\Raw_Data\myFiles",n)
   lf = open(local_filename, "wb")
   ftp.retrbinary("RETR " + n, lf.write, 1024)
   lf.close()
   print 'file written'
  cursor.execute("exec SP_my_Dailyfiles('n')")
  conn.close()
  lf.close()
  print 'sql executed'


ftp.quit()

stored proc:

ALTER PROCEDURE [dbo].[SP_my_Dailyfiles] 
    -- Add the parameters for the stored procedure here
@file VARCHAR(255)
-- Add the parameters for the stored procedure here

AS
BEGIN


IF EXISTS(SELECT * FROM sysobjects WHERE name = 'myinvoice')
DROP TABLE dbo.myinvoice
----------------------------------------------------------------------------------------------------
CREATE TABLE myinvoice(

     [Billing] varchar(255)
    ,[Order] varchar(45)
    ,[Item] varchar(255)
    ,[Quantity in pack] varchar(255)
    ,[Invoice] varchar(255)
    ,[Date] varchar(255)
    ,[Cost] varchar(255)
    ,[Quantity of pack] varchar(255)
    ,[Extended] varchar(255)
    ,[Type] varchar(25)
    ,[Date Due] varchar(255)

)

----------------------------------------------------------------------------------------------------            
DECLARE @SourceDirectory VARCHAR(255)
DECLARE @SourceFile VARCHAR(255)



EXEC (' BULK
    INSERT dbo.myinvoice 
    FROM ''D:\Raw_Data\myfile\'+@file+'''
    WITH
    (
        FIRSTROW = 1,
        FIELDTERMINATOR = '','',
        ROWTERMINATOR = ''0x0a''
    )'
)
-------------------------------------------------------------------------------------------------------------


INSERT INTO [Development].[dbo].[my_Dailyfiles](

     [Billing]
    ,[Order] 
    ,[Item] 
    ,[Quantity in pack] 
    ,[Invoice] 
    ,[Date]  
    ,[Cost] 
    ,[Quantity of pack] 
    ,[Extended] 
    ,[Type] 
    ,[Date Due] 
    ,[FileName] 
    ,[IMPORTEDDATE] 

    )
    SELECT

         replace([Billing], '"', '') 
          ,replace([Order], '"', '') 
          ,replace([Item], '"','') 
          ,replace([Quantity in pack],'"','') 
          ,replace([Invoice],'"','') 
          ,cast(replace([Date],'"','') as varchar(255)) as date
          ,replace([Cost],'"','')
          ,replace([Quantity of pack],'"','') 
          ,replace([Extended],'"','') 
          ,replace([Type],'"','') 
          ,cast(replace([Date Due],'"','') as varchar(255)) as date
          ,@file,
          GetDate()      


 FROM [myinvoice] WHERE [Bill to] <> ' '  and ndc != '"***********"'

Answer 1

I think the problem may be that you are closing the DB connection immediately after you execute the stored procedure, whilst still in the loop.

This means the second time around the loop, the DB connection is closed when you try to execute the SP. I would actually expect an error to be thrown the second around the loop.

The way I would structure this is something like:

conn = pyodbc.connect(...)
for n in nfiles:
    ...
    cursor = conn.cursor()
    cursor.execute("exec SP_my_Dailyfiles('n')")
    conn.commit()