Thanasis Naskos / Gstar Cluster Deploy

run.py 3.08 KB
edit raw blame history 
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
import paramiko, os, time, sys, subprocess

nodes = ['192.168.1.51','192.168.1.52','192.168.1.53','192.168.1.56','192.168.1.62','192.168.1.63','192.168.1.64','192.168.1.66','192.168.1.72','192.168.1.73','192.168.1.75','192.168.1.76','192.168.1.77','192.168.1.78','192.168.1.79','192.168.1.80']
if sys.argv[1] == 'start':
	if len(sys.argv) < 4:
		sys.exit("Wrong number of arguments! Please supply \"gstarnewFinal\", \"gstarnewLowMemory\" or \"gstar\" as the second argument and the name of the dataset as the third argument.")
	vertexID = subprocess.check_output('grep -m 1 \"vertex\" '+sys.argv[3]+'.txt | cut -d \" \" -f 2',shell=True)
	print "ID of the first vertex in dataset: " + vertexID
	os.system('sed -i "s/placeholder/'+str(vertexID).strip()+'/" /root/gstar_master/queryShortestDistance.txt')
	print "Starting G* Script. Starting Master..."
	os.system('cd /root/gstar_master; java -cp '+sys.argv[2]+'.jar gstar.Master -workers '+str(len(nodes))+' -port 2112  > /root/gstar_master/masteroutput 2>&1 &')
	time.sleep(5)
	i=0
	for node in nodes:
		ssh = paramiko.SSHClient()
		ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
		print("Starting worker: " + node) 
		ssh.connect(node, username='root', password='secretpw')
		ssh.exec_command('java -cp /root/gstar/'+sys.argv[2]+'.jar gstar.Worker '+str(i)+' '+str(int(2000+i))+' 192.168.1.74:2112 > /root/gstar/workeroutput 2>&1 &')
		ssh.close()
		i=i+1
	time.sleep(5)
	print "Terminal Starting..."
	os.system('cat /media/datasets/gstar/'+sys.argv[3]+'.txt | java -cp /root/gstar_master/'+sys.argv[2]+'.jar gstar.Terminal 2113 192.168.1.74:2112  > /root/gstar_master/output 2>&1 &')
elif sys.argv[1] == 'kill':
	os.system('sed -i \"/SingleSource/c\operator short@* = SingleSourceShortestDistanceOperator([], [0:1:5], placeholder, shortestattr)\" /root/gstar_master/queryShortestDistance.txt') #Reset the query
	for node in nodes:
		ssh = paramiko.SSHClient()
		ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
		print("Killing java and removing files from worker: " + node) 
		ssh.connect(node, username='root', password='secretpw')
		ssh.exec_command('killall -9 java; rm -r /root/sys')
		ssh.close()
	if os.path.exists('/root/gstar_master/sys'):
		os.system('rm -r /root/gstar_master/sys')
	os.system('killall -9 java')
elif sys.argv[1] == 'space':
	sumSpaceIndex = 0.0
	sumSpaceData = 0.0
	i = 0
	for node in nodes:
		ssh = paramiko.SSHClient()
		ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
		print("Calculating data and index file size for worker: " + node) 
		ssh.connect(node, username='root', password='secretpw')
		stdin, stdout, stderr = ssh.exec_command('awk \'NR==2\' /root/sys/'+str(i)+'_\!grafos.rst')
		line = stdout.read()
		sumSpaceIndex = sumSpaceIndex + float(line.split()[1])
		sumSpaceData = sumSpaceData + float(line.split()[2])
		ssh.close()
		i = i+1
	print("Total Index size: "+str(sumSpaceIndex)+" bytes/"+str(round(sumSpaceIndex/1024,2))+" KBytes/"+str(round(sumSpaceIndex/1024/1024,2))+" MBytes")
	print("Total Data size: " + str(sumSpaceData)+" bytes/"+str(round(sumSpaceData/1024,2))+" KBytes/"+str(round(sumSpaceData/1024/1024,2))+" MBytes")