""" filetransferbenchmark.py This module will run the file transfer benchmarking program. """ import time import random import sys, os if sys.platform == 'linux2': import pexpect from ftplib import FTP import csv import paramiko from configobj import ConfigObj config = ConfigObj("filetransferbenchmark.cfg") # done: Get the pExpect module working for linux ftp # todo: Get the Twisted ftpclient working for windows and linux # todo: Get the pscp client working for windows # done: Expand test cases # todo: Format output to csv class TestManager(object): def __init__(self): # Generate random data files of specific sizes based on config file section = config["Random Files"] filenames = section["filenames"] filesizes = section["filesizes"] if len(filenames)!= len(filesizes): print "In filetransferbenchmark.cfg, filenames and filesizes must have the same number of values!" sys.exit() self.bigranstr = str() self.bigranstrsize = 100000 i = 0 for filename in filenames: self.createrandomfile(filename, int(filesizes[i])) i += 1 self.bigranstr = str() # Get the test sets from the config file section = config["Tests"] reps = section['reps'] self.tests = list() i = 0 for filename in section['filenames']: self.tests.append((filename, int(reps[i]))) i += 1 print "self.tests =", self.tests def createrandomfile(self, filename, filesize): print "Entered createrandomfile()" if os.path.exists(filename) and os.path.getsize(filename) == filesize: print ' '*5,"%s already exists and is the correct size of %i"%(filename, filesize) else: if len(self.bigranstr) == 0: print "len(self.bigranstr) =", len(self.bigranstr) print "self.bigranstrsize =", self.bigranstrsize start = time.time() for i in range(self.bigranstrsize): self.bigranstr += chr(random.randint(0, 255)) print "It took %f seconds to create self.bigranstr"%(time.time() - start) start = time.time() f = open(filename, 'wb') if filesize < self.bigranstrsize: f.write(self.bigranstr[:filesize]) else: for i in range(int(filesize / self.bigranstrsize)): f.write(self.bigranstr) f.close() print "It took %f seconds to create %s a %i byte file."% \ (time.time()-start, filename,filesize) def run(self): """Method that sets up and executes the tests.""" csvf = file("Results %s.csv"% \ time.strftime("%Y%m%d-%H%M%S", time.localtime()), 'wb') self.csvwriter = csv.writer(csvf) self.csvwriter.writerow(['Description', 'Elapsed Time(s)', 'Filename', 'File size', \ 'Repetitions', 'Xfer Rate(MB/s)', 'Avg Xfer Time(s)']) self.output = file("Results %s.txt"% \ time.strftime("%Y%m%d-%H%M%S", time.localtime()), 'w') self.output.write("Test set started at %s.\n\n"%time.asctime()) self.output.write("Source path: %s\n"%config[sys.platform]["srcpath"]) self.output.write("Destination path: %s\n"%config["CDS"]["destpath"]) scpclient = ScpClient(config[sys.platform]["srcpath"], config["CDS"]["destpath"]) if sys.platform == 'linux2': ftpclient = FtpClient(config[sys.platform]["srcpath"], config["CDS"]["destpath"]) ftplibclient = FtplibFtpClient(config[sys.platform]["srcpath"], \ config["CDS"]["destpath"]) paramikosftpclient = ParamikoSftpClient(config[sys.platform]["srcpath"], \ config["CDS"]["destpath"]) for filename, reps in self.tests: print "scpclient sending %s for %i reps"%(filename, reps) self.execute(scpclient, filename, reps) if sys.platform == 'linux2': print "ftpclient sending %s for %i reps"%(filename, reps) self.execute(ftpclient, filename, reps) print "ftplibclient sending %s for %i reps"%(filename, reps) self.execute(ftplibclient, filename, reps) print "paramikosftpclient sending %s for %i reps"%(filename, reps) self.execute(paramikosftpclient, filename, reps) self.output.write("\nTest set ended at %s.\n\n"%time.asctime()) self.output.close() csvf.close() def execute(self, client, randomfile, repetitions): """Executes the test and formats the results""" filesize = os.path.getsize(randomfile) start = time.time() for i in range(repetitions): client.xfer(randomfile, i) elapsedtime = time.time() - start avgxfertime = elapsedtime / repetitions megabytesxfer = (repetitions * filesize)/1000000.0 xferrate = megabytesxfer / elapsedtime self.output.write("%s took %f seconds to write %s file %i times\n"% \ (client.descrip, elapsedtime, randomfile, repetitions)) self.output.write(" with a transfer rate of %f MB and avg xfer time of %f sec\n"% \ (xferrate, avgxfertime)) self.csvwriter.writerow([client.descrip, elapsedtime, randomfile, filesize, \ repetitions, xferrate, avgxfertime]) class Client(object): def __init__(self, srcpath, destpath, descrip): self.srcpath = srcpath self.destpath = destpath self.descrip = descrip def xfer(self, filename, repetition): pass class ScpClient(Client): def __init__(self, srcpath, destpath): print "Entered ScpClient.__init__()" Client.__init__(self, srcpath, destpath, "Using scp on %s"%sys.platform) section = config[sys.platform] self.scp = section["scp"] self.scp_arg = section["scp_arg"] if sys.platform == 'win32': self.scp_arg += ' ' + config["CDS"]["password"] print "scp =", self.scp def xfer(self, filename, repetition): "Transfer the file" splitfilename = os.path.splitext(filename) # Format for scp should be: scp -B /srcpath/filename username@host:/destpath src_arg = self.srcpath + filename dest_arg = config["CDS"]["username"] + "@" + config["CDS"]["host"] + ":" + \ self.destpath + 'scp_' + splitfilename[0] + "_" + str(repetition) + splitfilename[1] print "spawning:", self.scp, self.scp_arg, src_arg, dest_arg exit_code = os.spawnl(os.P_WAIT, self.scp, self.scp_arg, src_arg, dest_arg) #print exit_code class FtpClient(Client): def __init__(self, srcpath, destpath): print "Entered FtpClient.__init__()" Client.__init__(self, srcpath, destpath, "Using ftp on %s"%sys.platform) section = config[sys.platform] self.ftp = section["ftp"] self.ftp_arg = section["ftp_arg"] print "ftp =", self.ftp def xfer(self, filename, repetition): "Transfer the file" splitfilename = os.path.splitext(filename) # Format for scp should be: scp -B /srcpath/filename username@host:/destpath src_arg = self.srcpath + filename dest_arg = self.destpath + 'ftp_' + splitfilename[0] + "_" + str(repetition) + splitfilename[1] ftp_cmd = self.ftp + ' ' + self.ftp_arg + ' ' + config["CDS"]["host"] put_cmd = 'put ' + src_arg + ' ' + dest_arg #print "ftping:", ftp_cmd, src_arg, dest_arg if sys.platform == 'linux2': child = pexpect.spawn(ftp_cmd) child.expect('Name .*: ') child.sendline(config["CDS"]["username"]) child.expect('Password:') child.sendline(config['CDS']['password']) child.expect('ftp> ') child.sendline(put_cmd) child.expect('ftp> ') child.sendline('bye') else: print "windows ftp not implemented yet" class FtplibFtpClient(Client): def __init__(self, srcpath, destpath): print "Entered FtplibFtpClient.__init__()" Client.__init__(self, srcpath, destpath, "Using ftplib on %s"%sys.platform) section = config[sys.platform] self.ftp = section["ftp"] self.ftp_arg = section["ftp_arg"] def xfer(self, filename, repetition): "Transfer the file" splitfilename = os.path.splitext(filename) src_arg = self.srcpath + filename fp = file(src_arg,'rb') dest_arg = self.destpath + 'ftplib_' + splitfilename[0] + "_" + str(repetition) \ + splitfilename[1] ftp = FTP(config["CDS"]["host"], config["CDS"]["username"], config["CDS"]["password"]) ftp.storbinary('STOR ' + dest_arg, fp) ftp.sendcmd('QUIT') fp.close() class ParamikoSftpClient(Client): def __init__(self, srcpath, destpath): print "Entered ParamikoSftpClient.__init__()" Client.__init__(self, srcpath, destpath, "Using Paramiko sftp on %s"%sys.platform) self.client = paramiko.SSHClient() self.client.load_host_keys(config["CDS"]["host_keys"]) def xfer(self, filename, repetition): "Transfer the file" splitfilename = os.path.splitext(filename) src_arg = self.srcpath + filename dest_arg = self.destpath + 'sftp_' + splitfilename[0] + "_" + str(repetition) \ + splitfilename[1] self.client.connect(config["CDS"]["host"], username=config["CDS"]["username"], \ password=config["CDS"]["password"]) sftpclient = self.client.open_sftp() sftpclient.put(src_arg, dest_arg) sftpclient.close() if __name__ == "__main__": tm = TestManager() tm.run()