#!/usr/bin/python import sqlite3, re, sys, glob from problems import * from queries import * class ProgressBar: def __init__(self, Max): self.Max = Max self.percent = 0 self.i = 0 print '['+' '*100+'] 0%\r', sys.stdout.flush() def inc(self): self.i += 1 #p = ((self.i)/float(self.Max))*100 p = divmod(self.i*100,self.Max) if p[0] != self.percent: self.percent = p[0] print '['+'#'*int(p[0])+' '*(100-int(p[0]))+'] '+str(p[0])+'%\r', sys.stdout.flush() def get_point_id(x, y, z): c = db.cursor() c.execute(get_point_query % ("points",x,y,z)) res = c.fetchone() if not res: c.execute(insert_point_query % ("points", x,y,z)) res = c.lastrowid else: res = res[0] return res def check_defect_point(x,y,z,d): if d == 0: return False c = db.cursor() c.execute(get_point_query % ("defect",x,y,z)) res = c.fetchone() return False if not res else True def check_key(p, s, d, f, point_id): c = db.cursor() c.execute(check_key_query % (p,s,d,f,point_id)) res = c.fetchone() return False if res[0] >= 1 else True def fill_defect_table(file_name): print "Fill defect %s" % file_name c = db.cursor() c.executescript(table_defect_script) with open(file_name,"r") as defect: lines = defect.readlines() l = len(lines) pb = ProgressBar(l) for i in xrange(l): s = lines[i] spl = s.split() if (len(spl) > 0) and (re.match('^N[0-9]+$',spl[0])): c.execute(get_point_query % ("defect",spl[1],spl[2],spl[3])) res = c.fetchone() if not res: c.execute(insert_point_query % ("defect", spl[1],spl[2],spl[3])) pb.inc() print "\nDone" def process_file(file_name, _p, _s, _d, _f, _def): with open(file_name, "r") as equi: lines = equi.readlines() l = len(lines) pb = ProgressBar(l) props_cur = db.cursor() fields = "" for i in xrange(l): s = lines[i] spl = s.split() if len(spl) > 0: if spl[0] == 'NOEUD': s1 = lines[i+1] spl1 = s1.split() keys = spl[1:]+spl1 fields = ",".join(keys[3:]) if re.match('^N[0-9]+$',spl[0]): s1 = lines[i+1] spl1 = s1.split() m = spl[1:] + spl1 props = ",".join(m[3:]) if not check_defect_point(m[0],m[1],m[2],_def): p_id = get_point_id(m[0],m[1],m[2]) if check_key(_p,_s,_d,_f,p_id): query = insert_props_query % (fields,_p,_s,_d,_f,p_id,props) else: tf = fields.split(",") tp = ",".join(["=".join(x) for x in zip(tf,m[3:])]) query = update_props_query % (tp,_p,_s,_d,_f,p_id) props_cur.execute(query) pb.inc() db.commit() def get_defect_fnames(folder): l = glob.glob(folder + "/defect*") return l def process_problem(problem, schema, defect, phase): print "Process problem %s with schema %s for defect %s and phase %s" % \ (problem["description"],schema["folder"],defect["description"],phase["description"]) l = get_defect_fnames("/".join((problem["folder"],schema["folder"],defect["folder"]))) for i in l: if defect["append_matrix"] == 2 and phase["id"] == 0: print "Process defect file" process_file(i, problem["id"], schema["id"], defect["id"], phase["id"], defect["append_matrix"]) else: fill_defect_table(i) print "\nProcess properties file" file_name = "/".join((problem["folder"],schema["folder"],defect["folder"],"SIGM_NOEU_DEPL_%s.resu" % phase["name"])) print file_name process_file(file_name, problem["id"], schema["id"], defect["id"], phase["id"], defect["append_matrix"]) print "\nProcess another file" file_name = "/".join((problem["folder"],schema["folder"],defect["folder"],"EQUI_NOEU_SIGM_%s.resu" % phase["name"])) process_file(file_name, problem["id"], schema["id"], defect["id"], phase["id"], defect["append_matrix"]) db.cursor().execute(drop_defect_script) print "\nDone" db = sqlite3.connect("problems.db") db.executescript(table_props_script) db.executescript(table_points_script) process_problem(problems[0], schemas[0], defects[2], phases[1]) #for _problem in problems: #for _schema in _problem["schemas"]: #for _defect in defects: #for _phase in phases: #process_problem(_problem, schemas[_schema], _defect, _phase)