qdoc-bot - mozsearch

++ #!/usr/bin/python

++ # -*- coding: utf-8 -*-

++

++ # Copyright (C) 2015 The Qt Company Ltd.

++ # Contact: http://www.qt.io/licensing/

++ #

++ # You may use this file under the terms of the 3-clause BSD license.

++ # See the file LICENSE from this package for details.

++ #

++

++ # Checkouts have to have the remote: origin codereview-bot:qt/qtbase.git

++

++ # Run with --dir and --addr

++ # python qdoc-bot.py --dir /home/USER/dev/qt-qdoc/ --addr codereview-bot

++ # --dir is where the sources are checked out

++ # --addr codereview-bot

++ # With .ssh/config:

++ # Host codereview-bot

++ #    Hostname codereview.qt-project.org

++ #    Port 29418

++ #    User qt_docbot

++ #    IdentityFile /home/USER/.ssh/docbot_id_rsa

++

++

++

++ import os

++ import subprocess

++ import sys

++ import logging

++ import re

++

++ config = None # configuration object, it is initialized in main and passed to every worker process created by multiprocessing.Pool

++

++ #{

++     #"change": {

++         #"branch": "2.6",

++         #"id": "I765c9e165f9e7a5e0928401f5c5254397ad9be0b",

++         #"number": "33424",

++         #"owner": {

++             #"email": "Friedemann.Kleint@nokia.com",

++             #"name": "Friedemann Kleint"

++         #},

++         #"project": "qt-creator/qt-creator",

++         #"subject": "WIP: CDB: Fix some dumpers.",

++         #"url": "https://codereview.qt-project.org/33424"

++         #},

++     #"patchSet": {

++         #"number": "1",

++         #"ref": "refs/changes/24/33424/1",

++         #"revision": "c4dd516729ff8f9b7cb1e13d9c8f09560cf2c078",

++         #"uploader": {

++             #"email": "Friedemann.Kleint@nokia.com",

++             #"name": "Friedemann Kleint"

++         #}

++     #},

++     #"type": "patchset-created",

++     #"uploader": {

++         #"email": "Friedemann.Kleint@nokia.com",

++         #"name": "Friedemann Kleint"

++     #}

++ #}

++

++

++

++ def pasteKDE(data):

++     import httplib, urllib, json

++     params = urllib.urlencode({'paste_data': data,

++                                'paste_lang': 'text',

++                                'api_submit': True,

++                                'mode': 'json',

++                                'paste_project':'Qt',

++                                'paste_user': 'Qt Doc bot',

++                                'paste_private': 'yes',

++                                'paste_expire': 2629744 # around 1 month

++                               })

++     headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}

++     conn = httplib.HTTPConnection("paste.kde.org")

++     try:

++         conn.request("POST", "", params, headers)

++         response = json.loads(conn.getresponse().read())

++         logging.debug("Making a paste %s", response)

++         return "http://pastebin.kde.org/" + str(response['result']['id']) + "/" + str(response['result']['hash'])

++     finally:

++         conn.close()

++

++ #make sub-qdoc in src/tools

++ def run_qdoc(module_name):

++     output = []

++     try:

++         environment = os.environ

++         environment["CCACHE_BASEDIR"] =  os.getcwd()

++         environment["QT_HASH_SEED"] = "1234"

++         environment["QMAKEFLAGS"] = "CONFIG+=force_bootstrap"

++         for cmd in ["/bin/sh ./configure --opensource --confirm-license --release".split()

++                    , "make sub-src-qmake_all".split()

++                    , ["/bin/sh", "-c", "if [ -f src/tools/Makefile ] ; then make -Csrc/tools sub-qdoc; else make -Csrc sub-qdoc ; fi"] # build qdoc

++                    ]:

++             logging.debug("Executing command: %s", cmd)

++             try:

++                 proc = subprocess.Popen(cmd, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

++                 buildOutput = proc.communicate()[0]

++             except Exception, e:

++                 logging.error("Unknown error. Command execution failed: %s", str(e))

++                 raise e

++

++             if proc.returncode:

++                 raise subprocess.CalledProcessError(proc.returncode, cmd, buildOutput)

++

++         cmd = "make html_docs".split()

++         output.append(subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment).communicate()[1])

++         def smart_concatenate(r, x):

++             # concatenate multi-line comments

++             if x.startswith("    ["):

++                 r[-1] = r[-1] + x  # add comment

++             else:

++                 r.append(x)        # new error

++             return r

++         output[-1] = reduce(smart_concatenate, output[-1].split('\n'), [])

++         output[-1].sort()

++         output[-1] = "\n".join(output[-1])

++     except KeyError:

++         pass

++     return "\n".join(output)

++

++ #ssh codereview.qt-project.org gerrit review --project qt/qtbase --message "hello" 33219,44

++ def post_review(event, message, score):

++     project = event["change"]["project"]

++     sha1 = event["patchSet"]["revision"]

++     command = ["ssh", config.gerrit_address, "gerrit", "review", "--project", project,

++                "--code-review", str(score), "--tag", "autogenerated:qdoc"]

++     if len(message) > 0:

++         command += "--message", "\"" + message + "\"",

++     command += sha1,

++     logging.debug(command)

++     if not config.sandbox:

++         subprocess.check_call(command)

++     elif event["change"]["number"] == '48230':

++         subprocess.check_call(command)

++

++ def remove_moved_doc_errors(fixes, errors):

++     for fix in fixes:

++         pos = fix.find(":")

++         if pos >= 0:

++             begin = "+" + fix[1:pos]

++             end = fix[fix.find(":", pos+1):]

++             logging.debug(end)

++             for i in range(len(errors)):

++                 if errors[i].startswith(begin) and errors[i].endswith(end):

++                     del errors[i]

++                     break;

++     message = ""

++     for line in errors:

++         message += line + "\n "

++     return message

++

++ def review_output(event, output_no_patch, output_with_patch, score_on_negative_review):

++     try:

++         import difflib

++         project = event["change"]["project"]

++         patch_set = event["patchSet"]

++         url = event["change"]["url"]

++         subject = event["change"]["subject"]

++         new_error_count = output_with_patch.count("\n") - output_no_patch.count("\n")

++         if new_error_count == 0:

++             logging.info("NO DOC ERRORS: %s, %s", subject, url)

++             #post_review(event, "", 1)

++             return 0

++         logging.debug("DIFF for %s: %s", subject, url)

++         message = ""

++         score = 0

++         if new_error_count == 1:

++             message += "This change introduces one new documentation error."

++             score = score_on_negative_review

++         elif new_error_count > 1:

++             message += "This change introduces " + str(new_error_count) + " new documentation errors."

++             score = score_on_negative_review

++         elif new_error_count == -1:

++             message += "This change removes a documentation error. Thank you!"

++             score = 0

++         else:

++             message = "This change removes " + str(abs(new_error_count)) + " documentation errors. Thank you!"

++             score = 0

++

++         message += "\n\n "

++         fixes = []

++         errors = []

++         line_count = 0

++         for line in difflib.unified_diff(output_no_patch.split('\n'), output_with_patch.split('\n'), fromfile='before', tofile='after', n=0):

++             line_count += 1

++             if line_count > 2:

++                 if line[0] == "-":

++                     fixes.append(line)

++                 elif line[0] == "+":

++                     errors.append(line)

++         new_errors = remove_moved_doc_errors(fixes, errors)

++         if len(new_errors) > 0:

++             message += "New qdoc warnings:\n \n "

++             message += new_errors

++         message += "\n\nThis comment was generated by the qtdoc bot.\n\nPlease contact Jedrzej Nowacki (nierob) or Frederik Gladhorn (fregl) if you have questions or think the script gives you false positives."

++         logging.debug("Review result: %s", message)

++         post_review(event, message, score)

++         return score

++     except:

++         logging.error("Unexpected error: %s", sys.exc_info())

++

++

++ def fetch_change(project, patch_set, change_number):

++     try:

++         gerrit_url = config.gerrit_address + ":" + project

++         # the mainlines need to be updated, so that we keep the updated refs and fetches don't start from scratch every time

++         mainlines = subprocess.check_output(["git", "config", "remote.origin.fetch"]).rstrip()

++         fetch_cmd = ["git", "fetch", "-f", "origin", patch_set["ref"]+":refs/changes/"+change_number, mainlines]

++         num_tries = 0

++         while True:

++             try:

++                 p = subprocess.check_call(fetch_cmd)

++                 return 0

++

++             except subprocess.CalledProcessError, e:

++                 # Try to fetch again - one problem is that running git fetch will fail if run simultaneously

++                 num_tries += 1

++                 logging.warn("Fetching subprocess failed, trying again: %s", e)

++                 import time

++                 time.sleep(30)

++                 if num_tries > 11:

++                     raise e

++

++     except subprocess.CalledProcessError, e:

++         logging.error("Fetching subprocess failed too many times: %s", e)

++         return -1

++

++

++ #git fetch https://codereview.qt-project.org/p/qt/qtbase refs/changes/46/32446/6 && git checkout FETCH_HEAD

++ def process_event(event_string):

++     # load the event data

++     import time

++     start_time = time.time()

++     import json

++     import tempfile

++     import shutil

++

++     try:

++         event = json.loads(event_string)

++     except ValueError, e:

++         logging.error("JSON loading problem: %s\nJSON data: %s", e, event_string)

++         return -1

++

++     if event["type"] != "patchset-created":

++         return 0

++

++     # go to the right project checkout

++     project = event["change"]["project"]

++     patch_set = event["patchSet"]

++     change_number = event["change"]["number"]

++

++     module_name = project.split('/')[-1]

++     if module_name not in ["qtbase"]: # TODO for now we care only about qtbase

++         logging.info("IGNORED MODULE: %s %s", module_name, patch_set)

++         return 0

++

++     logging.info("MODULE NAME: %s %s", module_name, patch_set)

++

++     source_path = config.watcher_working_dir + "/" + project

++     try:

++         os.chdir(source_path)

++     except OSError, e:

++         logging.info("Unknown project: %s, trying to clone it", project)

++         tmp_dir = tempfile.mkdtemp(prefix="qdoc_code_")

++         os.chdir(tmp_dir)

++         try:

++             logging.debug("Clonning a new project (%s) to a tmp_dir (%s)", project, tmp_dir)

++             cmd_git_clone = ["git", "clone", "ssh://" + config.gerrit_address + "/" + project, "."]

++             subprocess.check_call(cmd_git_clone)

++         except Error, e:

++             logging.error("Could not clone a new project (%s) to a tmp_dir (%s), error message:", project, tmp_dir, e)

++             shutil.rmtree(tmp_dir)

++             return -1

++         try:

++             root_project_dir = config.watcher_working_dir + "/" + project.split('/')[:-1][0]

++             logging.debug("Creating destination folder (%s) for the new project (%s)", root_project_dir, project)

++             try:

++                 os.mkdir(root_project_dir)

++             except OSError, e:

++                 if e.errno != 17: # if error is different then "File exist"

++                     logging.error("Can not make project root directory: %s ", e)

++                     raise e

++             logging.debug("Moving the clone (%s) from tmp dir (%s) to the destination folder (%s)", project, tmp_dir, source_path)

++             os.rename(tmp_dir, source_path)

++         except Error, e:

++             logging.info("Initialization of a new project (%s) failed because it was initialized before, probably by an other process", project)

++             logging.debug("Exception caught during rename operation: %s", e)

++             shutil.rmtree(tmp_dir)

++         os.chdir(source_path)

++         logging.info("New project (%s) was initialized correctly", project)

++

++     if fetch_change(project, patch_set, change_number) != 0:

++         return -1

++

++     try:

++         #make own copy

++         tmp_dir = tempfile.mkdtemp(prefix="qdoc_")

++         logging.info("CREATING TMP DIR: %s", tmp_dir)

++         try:

++             os.chdir(tmp_dir)

++

++             cmd_git_clone = ["git", "clone", source_path, module_name]

++             subprocess.check_call(cmd_git_clone)

++

++             os.chdir(module_name)

++

++             cmd_git_reset = ["git", "checkout", event["patchSet"]["revision"], "-b", "tmp"]

++             subprocess.check_call(cmd_git_reset)

++

++             score_on_negative_review = -1

++             header = subprocess.check_output(["git", "log", '--pretty=oneline', '-n 1'])

++             if  (re.search(r'\bWIP\b', header) or header.find('***') >= 0):

++                 score_on_negative_review = 0 # Respect work in progress

++

++             try:

++                 output_with_patch = run_qdoc(module_name)

++             except subprocess.CalledProcessError, e:

++                 logging.debug("RUNNING QDOC FAILED: %s", e)

++                 msg = "Qt Doc Bot tried to build the change on Linux, without success. Most likely because of a bug in the patch. "

++                 msg += "Please verify that a clean build works."

++                 msg += "\n\nRunning command\n " + str(e.cmd) + "\nfailed."

++                 if e.output != None:

++                     link = pasteKDE(e.output)

++                     msg += "\n\nBuild log can be found here: "

++                     msg += link

++                 post_review(event, msg, -1)

++                 return -1

++

++             #reseting to parent and cleaning (especially needed for docs removal)

++             cmd_git_clean = ["git", "clean", "-fdxq"]

++             subprocess.check_call(cmd_git_clean)

++             cmd_git_reset = ["git", "reset", "--hard", "HEAD^"]

++             subprocess.check_call(cmd_git_reset)

++

++             output_no_patch = run_qdoc(module_name)

++

++             logging.info("DONE RUNNING QDOC")

++             return review_output(event, output_no_patch, output_with_patch, score_on_negative_review)

++

++         finally:

++             # TODO reuse tmp dir

++             logging.info("REMOVING TMP DIR: %s", tmp_dir)

++             shutil.rmtree(tmp_dir)

++             logging.debug("WORKER EXECUTION TIME: %s", time.time() - start_time)

++     except subprocess.CalledProcessError, e:

++         logging.error("QDoc sanity failed because of an internal error: %s", e)

++         return -1

++

++

++ def watcher():

++     while True:

++         try:

++             ssh = subprocess.Popen(["ssh", config.gerrit_address, "gerrit", "stream-events"], stdout=subprocess.PIPE)

++             import multiprocessing

++             logging.info("STARTING WATCHER")

++             def multiprocess_config_assign(c):

++                 config = c

++             processPool = multiprocessing.Pool(processes=config.workers_count,

++                                                initializer=multiprocess_config_assign,

++                                                initargs=(config,),

++                                                maxtasksperchild=5)

++             os.chdir(config.watcher_working_dir)

++             event_string = ssh.stdout.readline()

++             while len(event_string):

++                 try:

++                     processPool.apply_async(func=process_event, args=(event_string,))

++                 finally:

++                     #clean up working dir is it neccessery?

++                     os.chdir(config.watcher_working_dir)

++                 event_string = ssh.stdout.readline()

++             # we can get here if connection was broken somehow for example because of a timeout

++             logging.error("WATCHER RESTART because an empty response was received")

++             try:

++                 if ssh.returncode is None:

++                     # try to terminate the connection, it is better then wait for gc to do it.

++                     logging.warn("WATCHER received empty response but the connection was not closed, about to terminate it before restart")

++                     ssh.terminate()

++                 else:

++                     logging.warn("WATCHER was terminated with error code: %s", ssh.returncode)

++             except:

++                 pass

++         except:

++             logging.critical("WATCHER RESTART because of: %s", sys.exc_info())

++         finally:

++             # an error occurred let's restart in 15 sec

++             import time

++             time.sleep(15)

++             try:

++                 ssh.kill()

++             except: # swallow all errors, we do not care anymore

++                 pass

++

++

++ if __name__== "__main__":

++     class Config(object): pass

++     config = Config()

++     import argparse

++     parser = argparse.ArgumentParser(prog="QDoc bot",

++                    description='It listens to gerrit events and tries to catch all new documentation errors',

++                    epilog="QDoc bot takes into consideration different environment variables (like TMP or TEMP). It also may use ccache and icecc if configured correctly")

++     parser.add_argument('-j',

++                    type=int,

++                    dest="workers_count",

++                    default=None,

++                    help='Workers count, how many parallel builds can be done at the same time. Default is equals to CPU count.')

++     parser.add_argument('--addr',

++                    type=str,

++                    dest="gerrit_address",

++                    metavar="gerrit.example.org",

++                    default="codereview.qt-project.org",

++                    help='Address of the gerrit instance. Default is codereview.qt-project.org')

++     parser.add_argument('--dir',

++                    type=str,

++                    dest="watcher_working_dir",

++                    metavar="/home/user/dev/qdoc-watcher",

++                    required=True,

++                    help='Set working directory which contains all neccessery checkouts')

++     parser.add_argument('--log-level',

++                    type=str,

++                    dest="logging_level",

++                    choices=["debug", "info", "warning", "error", "critical"],

++                    default="INFO",

++                    help='Set verbose level')

++     parser.add_argument('--sandbox',

++                    dest="sandbox",

++                    action="store_true",

++                    default=False,

++                    help='The option should be used for debugging only. It simulates execution of the bot, post reviews are sent only for 33219 and 33224 changes')

++

++     args = parser.parse_args(namespace=config)

++     logging.basicConfig(format='(PID: %(process)d) %(asctime)s %(levelname)s: %(message)s', level=getattr(logging, config.logging_level.upper(), None))

++     watcher()

Revision control