#!/usr/bin/env python from collections import OrderedDict import subprocess, os, sys, getopt def choose_yes_no(): choice_made = False while not choice_made: yes = set(['yes', 'y', 'ye', '']) nope = set(['no', 'n']) choice = raw_input().lower() if choice in yes: choice_made = True return True elif choice in nope: choice_made = True return False else: print "Please respond with 'yes' or 'no'" def make_packages(these_packs, order): failed_packs = [] for my_pack in these_packs: # Go into the correct cmt directory my_commands = "cd " + order[my_pack] + "cmt/ &&" # Generate the makefile # my_commands += "cmt config && . ./setup.sh && " # Clean up the previous build my_commands += "make clean && " # Try a parallel build, then a normal build if it fails my_commands += "(make -j8 || make)" print my_commands return_code = subprocess.call(my_commands, shell=True) if return_code != 0: failed_packs.append(my_pack) else: message = (" Package " + my_pack + " has built ").center(80, "=") print "\n"+message+"\n" if len(failed_packs) == 0: print "Compiling has completed. We are now built completely." else: print "Packages failed to build:\n " + "\n ".join(failed_packs) # Turn off annoying pylint reminders about constant names # pylint: disable-msg=C0103 def main(argv): build_externals = False build_changed = False build_new = False build_merge = False targ_repo = None targ_b = None help_message = \ "Python scipt designed for rebuilding packages from a"\ " specific branch.\n"\ "Usage: check_branch.py \n\n"\ "Options:\n"\ "-b --branch : name of target branch to build\n"\ "-r --repo : name of target repo where branch is\n"\ " located\n"\ "-e --externals : include if externals should be compiled as well\n"\ "-c --changed : only rebuild packages that have changed from the\n"\ " master. NOTE: Use this if the official ICEDUST \n"\ " master is currently compiled\n"\ "-n --new : make and build a new local branch from the master\n"\ "-m --merge : pull this branch into a new copy of the master branch\n"\ "\n\n Example 1 : Compile from official icedust master\n"\ " check_branch.py --repo=official-icedust --branch=master\n\n"\ "\n Example 2 : Checkout a branch named my-icedust/test and build the\n"\ " packages changed compared to the\n"\ " official-icedust/master branch\n"\ " check_branch.py --repo=my-icedust --branch=test --changed\n"\ "\n Example 3 : Build a new branch named test from the master\n"\ " check_branch.py --branch=test --new\n"\ "\n Example 4 : Build the current state\n"\ "check_branch.py\n"\ "\n Example 5 : Checkout a branch and build the packages that have\n"\ " been changed in this checkout \n"\ "git checkout existing_branch\n"\ "check_branch.py --changed\n" try: opts, _ = getopt.getopt(argv, "hb:r:ecnm",\ ["branch=", "repo=", "externals", "changed", "new", "merge"]) except getopt.GetoptError: print help_message sys.exit(2) for opt, arg in opts: if opt == '-h': print help_message sys.exit() elif opt in ("-b", "--branch"): targ_b = arg elif opt in ("-r", "--repo"): targ_repo = arg elif opt in ("-e", "--externals"): build_externals = True elif opt in ("-c", "--changed"): build_changed = True elif opt in ("-n", "--new"): build_new = True targ_repo = "" elif opt in ("-m", "--merge"): build_merge = True # Remember where we are this_dir = os.path.expandvars("$PWD")+"/" # Change into where git will see some differences project_dir = os.path.expandvars("$ICEDUST_HOME")+"/OfflineProject/packages" cmt_dir = os.path.expandvars("$ICEDUST_HOME")+"/OfflineProject/cmt" # Check if we are building a branch we are not currently on if (targ_repo != None) and (targ_b != None): # Record where the full branch is full_branch = targ_repo + "/" + targ_b # Move to where the repo is commands = "cd "+project_dir+"; " if build_new or build_merge: # If a new branch is built, lets name it print "We are building a new branch from " + full_branch # Checkout the target branch as a new branch if build_new and not build_merge: new_b = targ_b print "Creating a new branch named " + new_b +\ " from the official master" commands += "git checkout -b "+new_b+" official-icedust/master;" # print "Sucessfully created branch " + new_b else: # Pull in the changes new_b = targ_repo + "_"+targ_b + "_check" print "Pulling in the desired changes from the target branch"\ " to the new branch" commands += "git checkout -b "+new_b+" official-icedust/master;" commands += "git pull --no-edit " + targ_repo + " " + targ_b # print "Sucessfully created the after-merge state branch " + new_b else: # We will be looking at it from detached state print "We are checking out " + full_branch commands += "git checkout " + full_branch print "Sucessfully checked out " + full_branch subprocess.check_call(commands, shell=True) # Return ordered list of packages to update commands = "cd " + cmt_dir +"; " # Pull out the package name and its location, i.e. words 2 and 4 of all # lines after the one with Selection as the second word commands += "cmt show uses | awk '{ print $2 \" \" $4 }'\ | awk '/Selection/{flag=1;next}flag'" # Build a list with each line as an entry update_order = subprocess.check_output(commands, shell=True).split("\n") # Ignore the last line update_order = update_order[:-1] if not build_externals: # Ignore all packages in EXTERNALS update_order = [up for up in update_order if up.endswith("packages)")\ and not up.startswith("SimMARS")\ and not up.startswith("SimPHITS")] # Extract the package names as the first word of each element in the list packages = [package.split(" ")[0] for package in update_order] # Extract the root directory of the package as the second word in the list. # Also trim the first and last characters from each package location package_dirs = [package.split(" ")[1][1:-1] for package in update_order] # Add the package name to the package directory package_dirs = [direct+"/"+pack+"/" for direct, pack \ in zip(package_dirs, packages)] # Build an ordered dictionary that maps from package name to package # location package_order = OrderedDict(zip(packages, package_dirs)) if build_changed: # Find out which packages have been updated. Do this by looking for # the names of the changed packages in the last 20 git commands. i = 1 packs_found = False while (i < 21) and not packs_found: # Go into the location of the git repository commands = "cd " + project_dir +";" # Find out which packages have been changed commands += "git diff --name-only HEAD@{0}..HEAD@{"+str(i)+"} |"+\ " cut -f1 -d/ | sort -u" # Collect output as list of files that have been changed to_make = subprocess.check_output(commands, shell=True).split("\n") # Ignore the last line to_make = to_make[:-1] # Check if there are any actual files if len(to_make) > 0: packs_found = True i += 1 # Rebuild the packages that have been changed if packs_found: print "Found changed files "+str(i)+" git commands ago" # Check if there are any packages that have been updated that are # not currently added to the working package list missing_packages = set(to_make) - set(packages) if len(missing_packages) != 0: print "Packages are missing. Check if they have been deleted." for miss in missing_packages: print " Has "+miss+" been deleted in this change?" if choose_yes_no(): print " Package "+miss+" will not be built" to_make.remove(miss) missing_packages = set(to_make) - set(packages) assert len(missing_packages) == 0,\ "\n \n Error: Package(s) " +" ".join(missing_packages)+\ " do not exist yet and have not been removed."+\ "For each, try: \n cd "+ project_dir+\ " \n git icedust-add \n" # Sort the packages by the order CMT states to_make = sorted(to_make, key=lambda word: packages.index(word)) else: print "No changes found in the previous " + str(i-1) +\ " git commands" else: to_make = packages # Let the user know print "Packages are: \n " + "\n ".join(to_make) # Ask them if they want to build the listed packages print "Build these packages? Enter [yes/no]" if choose_yes_no(): # If yes, loop over all the packages in the list make_packages(to_make, package_order) print "Revert back to the build of the master?" if choose_yes_no(): # If yes, lets switch back to the master and undo all the builds we # did commands = "cd " + project_dir+";" commands += "git checkout official-icedust/master" subprocess.check_call(commands, shell=True) make_packages(to_make, package_order) subprocess.call("cd "+ this_dir, shell=True) if __name__ == "__main__": main(sys.argv[1:])