The following code works well 99% of the time.
However, when I use it to copy a directory with a large number of files (some of the individual files are also large) it hangs (no exceptions thrown) and any further FTP requests hang until I recycle the IIS 7.5 application pool that the code is running under. (This is used in a web-based file browser.)
It does not hang on the same file each time and actually lets me completely copy the directory once successfully, but then if I try and do it again it hangs after only copying some of the files and sub-directories.
My question is, can anyone see an obvious problem with the code? Is there a connection object that isn’t closed properly or something?
Incidentally, I have tried (in the FtpCopyFile method) flushing and closing the “uploadStream” object as well as instantiating the FtpWebResponse object and subsequently closing it. Neither of those changes made any difference.
If there is nothing obvious with the code, can anyone recommend a method for tracking down the problem? Since no exception is thrown and I can’t find anything in the server logs (at least the ones I know to look at), I am at a loss.
Any help would be greatly appreciated!
fodder
public string FtpCopy(string fromUrl, string toUrl, bool isDirectory)
{
string copyResult = "";
// COPY ENTIRE DIRECTORY
if (isDirectory) {
// MAKE SURE TOP DIRECTORY IS CREATED
if (!FtpDirectoryExists(toUrl)) { copyResult += FtpMakeDirectory(toUrl); }
// ITERATE TROUGH ALL FILES AND FOLDERS AND COPY TO LIVE LOCATION
Dictionary<string,Dictionary<string,string>> newItems = FtpRecursiveFileList(fromUrl);
foreach (KeyValuePair<string,Dictionary<string,string>> item in newItems) {
string currentFromUrl = item.Key;
string currentToUrl = currentFromUrl.Replace(fromUrl, toUrl);
if(item.Value["isdirectory"] == "true") { copyResult += FtpMakeDirectory(currentToUrl); }
else { copyResult += FtpCopyFile(currentFromUrl, currentToUrl); }
}
// COPY SINGLE FILE
} else { copyResult = FtpCopyFile(fromUrl, toUrl); }
string returnString = "";
if (copyResult == "") { returnString = "Success"; }
else { returnString = "Error: " + copyResult; }
return returnString;
}
private string FtpMakeDirectory(string url) {
string returnString = "";
// PARSE URL
url = url.TrimEnd('/') + "/";
string[] urlPath = Jbu.Util.UrlToStringArray(url, FTP_PATH_PREFIX);
string currentPath = FTP_PATH_PREFIX + urlPath[0];
// LOOP THROUGH EACH DIRECTORY LEVEL OF PATH
for (int i = 1; i < (urlPath.Length - 1); i++) {
currentPath = currentPath + "/" + urlPath[i];
string[] currentFiles = FtpListDirectoryArray(currentPath);
bool found = false;
if (currentFiles != null) {
// LOOK IN CURRENT DIRECTORY FOR DIRECTORY THAT HAS SAME NAME AS NEXT LOOP'S DIRECTORY
for (int j = 0; j < currentFiles.Length; j++) {
if (currentFiles[j] == urlPath[i + 1]) { found = true; }
}
}
// IF NAME NOT FOUND, CREATE DIRECTORY
if(!found) { returnString += FtpResponseAsString(CreateFtpRequest(currentPath + "/" + urlPath[i + 1], "makedirectory")); }
}
return returnString;
}
private string FtpCopyFile(string fromUrl, string toUrl)
{
string returnString = "";
try {
// GET FILE TO BE COPIED
FtpWebRequest ftpDownloadRequest = CreateFtpRequest(fromUrl, "downloadfile");
System.Net.FtpWebResponse downloadResponse = (System.Net.FtpWebResponse)ftpDownloadRequest.GetResponse();
Stream ftpDownloadStream = downloadResponse.GetResponseStream();
byte[] fileByteArray = Jbu.Util.StreamToByteArray(ftpDownloadStream);
ftpDownloadStream.Close();
// CREATE DIRECTORY, IF NEEDED
string containingDirectory = toUrl.Substring(0,toUrl.LastIndexOf('/'));
if (!FtpDirectoryExists(containingDirectory)) { returnString += FtpMakeDirectory(containingDirectory); }
// UPLOAD FILE TO NEW LOCATION
FtpWebRequest ftpUploadRequest = CreateFtpRequest(toUrl, "uploadfile");
ftpUploadRequest.ContentLength = fileByteArray.Length;
using (Stream uploadStream = ftpUploadRequest.GetRequestStream()) { uploadStream.Write(fileByteArray, 0, fileByteArray.Length); }
} catch (Exception ex) { returnString += "Error: " + ex.ToString(); }
return returnString;
}
private FtpWebRequest CreateFtpRequest(string url, string method)
{
// CREATE REQUEST OBJECT
ServicePointManager.ServerCertificateValidationCallback = (Object obj, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) => (certificate.Subject.Contains("CN=" + Jbu.Constant.FTP_CERT_DOMAIN));
FtpWebRequest ftpRequest = (FtpWebRequest)FtpWebRequest.Create(new Uri(url));
ftpRequest.EnableSsl = true;
ftpRequest.Credentials = new NetworkCredential(Jbu.Constant.FTP_USER, Jbu.Constant.FTP_PASSWORD);
ftpRequest.UseBinary = true;
ftpRequest.KeepAlive = false;
// SET METHOD
switch(method) {
case "listdirectory": ftpRequest.Method = WebRequestMethods.Ftp.ListDirectory; break;
case "listdirectorydetails": ftpRequest.Method = WebRequestMethods.Ftp.ListDirectoryDetails; break;
case "makedirectory": ftpRequest.Method = WebRequestMethods.Ftp.MakeDirectory; break;
case "removedirectory": ftpRequest.Method = WebRequestMethods.Ftp.RemoveDirectory; break;
case "downloadfile": ftpRequest.Method = WebRequestMethods.Ftp.DownloadFile; break;
case "uploadfile": ftpRequest.Method = WebRequestMethods.Ftp.UploadFile; break;
case "deletefile": ftpRequest.Method = WebRequestMethods.Ftp.DeleteFile; break;
case "getdatetimestamp": ftpRequest.Method = WebRequestMethods.Ftp.GetDateTimestamp; break;
default: break;
}
return ftpRequest;
}
private bool FtpDirectoryExists(string url)
{
bool dirExists = true;
try {
FtpWebRequest ftpRequest = CreateFtpRequest(url + "/", "listdirectory");
FtpWebResponse ftpResponse = (FtpWebResponse)ftpRequest.GetResponse();
} catch { dirExists = false; }
return dirExists;
}
private Dictionary<string,Dictionary<string,string>> FtpRecursiveFileList(string url)
{
Dictionary<string,Dictionary<string,string>> returnList = new Dictionary<string,Dictionary<string,string>>();
List<string> files = new List<string>();
Queue<string> folders = new Queue<string>();
folders.Enqueue(url);
while (folders.Count > 0) {
string fld = folders.Dequeue();
Dictionary<string,Dictionary<string,string>> newItems = FtpListDirectoryDetailsArray(fld);
foreach(KeyValuePair<string,Dictionary<string,string>> item in newItems) {
returnList.Add(fld + "/" + item.Key, item.Value);
if(item.Value["isdirectory"] == "true") {
folders.Enqueue(fld + "/" + item.Key);
}
}
}
return returnList;
}
private string[] FtpListDirectoryArray(string ftpPath)
{
FtpWebRequest ftpRequest = CreateFtpRequest(ftpPath, "listdirectory");
List<string> items = new List<string>();
try {
FtpWebResponse ftpResponse = (FtpWebResponse)ftpRequest.GetResponse();
Stream responseStream = ftpResponse.GetResponseStream();
using (StreamReader responseReader = new StreamReader(responseStream)) {
string line;
while ((line = responseReader.ReadLine()) != null) { items.Add(line); }
}
} catch { return null; }
string[] itemData = new string[items.Count];
for (int i = 0; i < items.Count; i++) { itemData[i] = items[i]; }
return itemData;
}
private Dictionary<string,Dictionary<string,string>> FtpListDirectoryDetailsArray(string ftpPath)
{
Dictionary<string,Dictionary<string,string>> items = new Dictionary<string,Dictionary<string,string>>();
FtpWebRequest ftpRequest = CreateFtpRequest(ftpPath, "listdirectorydetails");
try {
FtpWebResponse ftpResponse = (FtpWebResponse)ftpRequest.GetResponse();
Stream responseStream = ftpResponse.GetResponseStream();
using (StreamReader responseReader = new StreamReader(responseStream)) {
string line;
while ((line = responseReader.ReadLine()) != null) {
Dictionary<string,string> item = new Dictionary<string,string>();
line = System.Text.RegularExpressions.Regex.Replace(line, @"\s+", " "); // REMOVE EXTRA SPACES
string[] itemDetails = line.Split(' ');
item.Add("datetime", itemDetails[0] + " " + itemDetails[1]);
// FOLDERS
if (itemDetails[2] == "<DIR>") {
item.Add("isdirectory", "true");
item.Add("size", "-1");
item.Add("name", itemDetails[3]);
} else {
item.Add("isdirectory", "false");
item.Add("size", itemDetails[2]);
item.Add("name", itemDetails[3]);
}
items.Add(itemDetails[3], item);
}
}
// IF DIRECTORY DOES NOT EXIST, RETURN EMPTY DICT
} catch {};
return items;
}
private string FtpResponseAsString(FtpWebRequest ftpRequest)
{
try {
FtpWebResponse ftpResponse = (FtpWebResponse)ftpRequest.GetResponse();
Stream responseStream = ftpResponse.GetResponseStream();
StreamReader responseReader = new StreamReader(responseStream);
return responseReader.ReadToEnd();
} catch (Exception ex) { return "Error: " + ftpRequest.RequestUri + "\n\n" + ex.ToString(); }
}
Finally found the problem!
“FtpWebRequest.KeepAlive = false;” does work, but it takes a little time to clear out closed connections.
So what was happening is that my maximum number of connections was being hit. (Apparently there is a .NET max, as well, because my maxconnections in IIS was already set very high.)
Adding this line to the CreateFtpConnection method solves the issue by giving IIS enough time to close the old connections and make room for more: ftpRequest.ServicePoint.ConnectionLimit = 1000;
Your specific mileage will most likely vary based on your file sizes but hopefully this will help someone.
For the record, here is what CreateFtpConnection looks like now: