git-qt-merge-mainlines

#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2015 The Qt Company Ltd.

# Contact: http://www.qt.io/licensing/

# You may use this file under the terms of the 3-clause BSD license.

# See the file LICENSE from this package for details.

from __future__ import print_function

import os

import sys

import logging

import subprocess

import re

import warnings

fnull = open(os.devnull, "w")

class colors:

    HEADER = '\033[95m'

    BLUE = '\033[94m'

    GREEN = '\033[92m'

    WARNING = '\033[93m'

    FAIL = '\033[91m'

    ENDC = '\033[0m'

class BlameParser:

    __begin = ('<' * 7, '=' * 7)

    __end_map = {'merge': '=', 'diff3': '|'}

    # One example line to parse (result from git blame -c --date=short):

    # "ddd8ab31	(Andras Becsi	2014-06-13 18:17:07 +0200	414)bool GLSurface::InitializeOneOffInternal()"

    __matcher = re.compile(r"([0-9a-f]+)\s+\((.*?)\s+\d+\)(.*)").match

    def __init__(self):

        self.__end = (self.__from_git_config('merge.conflictstyle', 'merge', self.__end_map) * 7, '>' * 7)

    def __from_git_config(self, parameter, default, mapping):

        try:

            value = subprocess.check_output(['git', 'config', parameter]).decode('utf-8').strip()

        except subprocess.CalledProcessError:

            # When the value is unset

            value = ''

        try:

            return mapping[value or default]

        except KeyError:

            raise ValueError('Unknown git config', parameter, value)

    def __parse_blame_line(self, codeline):

        # Match a "git blame -c --date=short" output line

        m = self.__matcher(codeline)

        if m:

            return m.group(1, 2, 3)

        raise ValueError("Failed to parse blame line: %s" % (codeline))

    def __get_blame_results(self, conflict):

        cmd_git_blame = ["git", "blame", "-c", "--date=short", conflict]

        blame_output = subprocess.check_output(cmd_git_blame).decode('utf-8')

        return blame_output.split('\n')

    def parse(self, conflict):

        results, scan = {}, False

        for line in self.__get_blame_results(conflict):

            if not line.strip():

                continue

            sha1, info, code = self.__parse_blame_line(line)

            if any(code.startswith(b) for b in self.__begin):

                scan = True

                continue

            if any(code.startswith(e) for e in self.__end):

                scan = False

            if scan:

                results[sha1] = info

        return results.items()

# Try to find related commits for the conflicts

def check_conflicts(module, config, from_ref, conflicts):

    results = ''

    blameparser = BlameParser()

    for conflict in conflicts:

        changes = blameparser.parse(conflict)

        results += conflict + ':\n\t' + '\n\t'.join('%s %s' %c for c in changes) + '\n'

    return results

# Replace all occurrences of searchExp in one file

def replaceInFile(file, searchExp, replaceExp):

    import fileinput

    for line in fileinput.input(file, inplace=1):

        line = re.sub(searchExp, replaceExp, line)

        sys.stdout.write(line)

# Reset module to destination branch

def reset_module(module, config):

    opts = {}

    opts['to'] = config.branch_to

    try:

        git_checkout = "git checkout %(to)s" %opts

        subprocess.check_call(git_checkout.split(), stdout=fnull)

        git_reset = "git reset --hard origin/%(to)s" %opts

        subprocess.check_call(git_reset.split(), stdout=fnull, stderr=fnull)

    except Exception as e:

        logging.error("Git reset failed: %s", str(e))

        return False

    return True

# Run git merge for one module

def do_merge(module, config, from_ref):

    try:

        cmd_git_merge = ["git", "merge", "-m", \

                         "Merge remote-tracking branch \'origin/" + config.branch_from + "\' into " + config.branch_to, \

                         from_ref, "--no-edit", "--no-ff"]

        ret = subprocess.call(cmd_git_merge, stdout=fnull)

        if ret != 0 and config.mergetool:

            print("Starting mergetool")

            ret = subprocess.call(['git', 'mergetool', '-y'])

            if ret == 0:

                ret = subprocess.call(['git', 'commit', '--no-edit'], stdout=fnull)

        if ret != 0:

            print("Module %s failed to merge, manual merge needed." % module)

            return False

        # Make sure we get a change-id as git merge doesn't generate it for us

        git_amend = "git commit --amend --no-edit"

        subprocess.check_call(git_amend.split(), stdout=fnull)

    except Exception as e:

        logging.error("Merge failed: %s", str(e))

        return False

    return True

def wip_merge(module, config, from_ref):

    conflicts = []

    try:

        cmd_git_status = ["git", "status", "--porcelain"]

        status_output = subprocess.check_output(cmd_git_status).decode('utf-8').strip()

        status_count = 0

        if status_output:

            lines = status_output.split('\n')

            status_count = len(lines)

            for line in lines:

                try: mode, file = line.split()

                except (ValueError, TypeError): pass # not exactly two tokens

                else:

                    if 'U' in mode:

                        print("conflict: ", file)

                        conflicts.append(file)

        if not conflicts:

            print("Didn't find any conflict. Need to check carefully.")

            return False

        cmd_git_commit = ["git", "commit", "-a", "-m",

                          "WIP: Merge remote-tracking branch \'origin/" + config.branch_from + "\' into " + config.branch_to + "\n"

                          + '\nConflicts:\n\t' + '\n\t'.join(conflicts) + '\n']

        ret = subprocess.call(cmd_git_commit, stdout=fnull)

        if ret != 0:

            print("Failed to do \"git commit -a\".")

            return False

        # Make sure we get a change-id as git merge doesn't generate it for us

        git_amend = "git commit --amend --no-edit"

        subprocess.check_call(git_amend.split(), stdout=fnull)

        related = check_conflicts(module, config, from_ref, conflicts)

        print("Related: ", related)

    except Exception as e:

        logging.error("Create WIP change failed: %s", str(e))

        return False

    return True

# Change the version number in the .qmake.conf file to a given string

def update_qmake_conf(module, version):

    fileName = ".qmake.conf"

    replaceInFile(fileName, "5\..\..", version)

    cmd_git_status = "git status".split()

    status = subprocess.check_output(cmd_git_status).decode('utf-8')

    print(status)

    if status.find(fileName) < 0:

        return False

    cmd_git_add = "git add .qmake.conf".split()

    subprocess.check_call(cmd_git_add, stdout=fnull)

    cmd_git_commit = ['git', 'commit', '-m', 'Update module version to %s' % version]

    subprocess.check_call(cmd_git_commit, stdout=fnull)

    return True

# Run git push to the destination branch for one module

def push_gerrit(module, config):

    subprocess.call(["git", "show"])

    print('\n"%s" was successfully updated and merged' % module)

    # fix for python 2.x compatibility

    try: input = raw_input

    except NameError: pass

    confirm = input('Push merge? [Y/n] ')

    if not confirm or confirm.startswith(('y', 'Y')):

        output = subprocess.check_output(('git push gerrit HEAD:refs/for/%s' % config.branch_to).split()).decode('utf-8')

        #url = re.search("https.*", output).group(0).strip()

        #print(url)

        #GERRIT_CHANGE_ID=$(echo "$GIT_PUSH" | sed 's/.*https:\/\/codereview.qt-project.org\/\(.*\)/\;tx;d;:x')

        #echo "ChangeId: $GERRIT_CHANGE_ID"

        # if we had gerrit 2.4, we could do this:

        #for reviewer in

        #   sh codereview.qt-project.org gerrit set-reviewers --add jedrzej.nowacki@digia.com 49938

    else:

        print(colors.WARNING + "Merge not pushed." + colors.ENDC)

# Perform merge and push it for one module

def merge(module, config, from_ref):

    push_required = False

    if config.merge or config.wip:

        if not do_merge(module, config, from_ref):

            return False

        push_required = True

    if config.version:

        print('Updating .qmake.conf for', module)

        if update_qmake_conf(module, config.version):

            push_required = True

    if push_required:

        push_gerrit(module, config)

    return True

# Create WIP change when having conflicts

def create_wip(module, config, from_ref):

    if config.wip:

        if not wip_merge(module, config, from_ref):

            return False

        push_gerrit(module, config)

    return True

def get_module_sha_from_super(module, branch):

    result = ""

    cmd_git_lstree = ["git", "ls-tree", "origin/" + branch, module]

    lstree_output = subprocess.check_output(cmd_git_lstree).decode('utf-8').strip()

    lstree_outputs = lstree_output.split()

    if len(lstree_outputs) == 4 and lstree_outputs[3] == module:

        result = lstree_outputs[2].strip()

    return result

# Iterate over all modules (either default or passed with -m option)

def process_modules(config):

    if config.list_modules:

        print("Modules: ", config.modules)

        return

    cmd_git_fetch = ["git", "fetch"]

    subprocess.check_call(cmd_git_fetch, stdout=fnull, stderr=fnull)

    manual_merges = []

    for module in config.modules.split():

        print("\nModule: " + colors.GREEN + module + colors.ENDC)

        if not os.path.isdir(module):

            print("Directory '%s' does not exist. Skipping..." % module)

            continue

        try:

            os.chdir(module)

            try:

                cmd_git_fetch = ["git", "fetch"]

                subprocess.check_call(cmd_git_fetch, stdout=fnull, stderr=fnull)

                to_string = "origin/" + config.branch_to

                from_string = "origin/" + config.branch_from

                if config.super:

                    os.chdir("..")

                    from_sha1 = get_module_sha_from_super(module, config.branch_from)

                    if from_sha1:

                        from_string = from_sha1

                    else:

                        print("Unable to obtain sha1 from super module for module ", module, "\n")

                    os.chdir(module)

                print("Merge from", from_string, "to", to_string, ":")

                cmd_git_cherry = ["git", "cherry", to_string, from_string, "-v"]

                cherry_output = subprocess.check_output(cmd_git_cherry).decode('utf-8').strip()

                change_count = 0

                if cherry_output:

                    change_count = len(cherry_output.split('\n'))

                if config.status:

                    print(cherry_output)

                    print(colors.GREEN, change_count, colors.ENDC, "patches to be merged in", module, "\n")

                if config.reset or config.merge or config.version or config.wip:

                    reset_module(module, config)

                if change_count > 0:

                    if not merge(module, config, from_string):

                        manual_merges.append(module)

                        if config.wip:

                            create_wip(module, config, from_string)

            except Exception as e:

                logging.error("Command execution failed: %s", str(e))

                import traceback

                traceback.print_exc(file=sys.stdout)

            finally:

                os.chdir("..")

        except Exception as e:

            logging.error("Changing current dir failed: %s", str(e))

            import traceback

            traceback.print_exc(file=sys.stdout)

    if manual_merges:

        print("Modules failed to merge: ", manual_merges)

# get a list of submodules and a list of submodules that are not checked out

def get_submodules():

    git_submodule_status = subprocess.check_output('git submodule status'.split(' ')).strip()

    modules = []

    modules_not_checked_out = []

    for line in git_submodule_status.split('\n'):

        module = line.strip().split(' ')[1]

        if line.startswith('-'):

            modules_not_checked_out.append(module)

            print('WARNING:', module, 'is not checked out')

        elif module in ('qtqa', 'qtrepotools'):

            print('skipping', module)

        else:

            modules.append(module)

    return modules, modules_not_checked_out

if __name__== "__main__":

    import argparse

    parser = argparse.ArgumentParser(prog="git-qt-merge-branches",

        description="Merge branches for the Qt Project",

        epilog="""

Run from within a work tree of the qt5 super-module, with all relevant

sub-modules checked out (e.g. using qt5/init-repository).

""")

    group = parser.add_argument_group("Primary action", "Chose one of:")

    group.add_argument('-s', '--status', action="store_true",

                       help='show the status (which patches will be merged)')

    group.add_argument('-d', '--merge', action="store_true",

                       help='do the merge')

    parser.add_argument('-u', '--super', action="store_true",

                        help='obtain sha1s from supermodule instead of using branch tips')

    parser.add_argument('-m', '--modules',

                        help='override the list of modules (eg. -m "qtbase qtdeclarative")')

    group.add_argument('-l', '--list-modules', action="store_true",

                       help='list the modules to be merged and exit')

    group.add_argument('--reset', action="store_true",

                       help='reset to origin/to_branch. this is implicit in the merge command')

    parser.add_argument('-f', '--branch-from', default='stable',

                        help='from which branch to merge')

    parser.add_argument('-t', '--branch-to', default='dev',

                        help='the target branch')

    group.add_argument('-v', '--version', default='',

                       help='set version in .qmake.conf to given version string')

    parser.add_argument('--mergetool', action="store_true",

                        help='run mergetool for conflicts')

    parser.add_argument('-w', '--wip', action="store_true",

                        help='create WIP change when conflicts arise')

    config = parser.parse_args()

    print("Qt Project merge tool\n")

    default_modules, modules_not_checked_out = get_submodules()

    if config.modules is None:

        config.modules = ' '.join(default_modules)

    print("Submodules: ", default_modules)

    print("Ignored submodules: ", modules_not_checked_out)

    logging.basicConfig(format='%(levelname)s: %(message)s')

    if config.status or config.merge or config.reset or config.list_modules or config.version or config.wip:

        process_modules(config)

    else:

        parser.print_help()

    print("Submodules that are not checked out and are ignored:\n",

          ' '.join(modules_not_checked_out))

Revision control

Copy as Markdown