Revision control
+ ############################################################################
+ ##
+ ## Copyright (C) 2021 The Qt Company Ltd.
+ ## Contact: https://www.qt.io/licensing/
+ ##
+ ## This file is part of the qtqa module of the Qt Toolkit.
+ ##
+ ## $QT_BEGIN_LICENSE:LGPL$
+ ## Commercial License Usage
+ ## Licensees holding valid commercial Qt licenses may use this file in
+ ## accordance with the commercial license agreement provided with the
+ ## Software or, alternatively, in accordance with the terms contained in
+ ## a written agreement between you and The Qt Company. For licensing terms
+ ## and conditions see https://www.qt.io/terms-conditions. For further
+ ## information use the contact form at https://www.qt.io/contact-us.
+ ##
+ ## GNU Lesser General Public License Usage
+ ## Alternatively, this file may be used under the terms of the GNU Lesser
+ ## General Public License version 3 as published by the Free Software
+ ## Foundation and appearing in the file LICENSE.LGPL3 included in the
+ ## packaging of this file. Please review the following information to
+ ## ensure the GNU Lesser General Public License version 3 requirements
+ ## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+ ##
+ ## GNU General Public License Usage
+ ## Alternatively, this file may be used under the terms of the GNU
+ ## General Public License version 2.0 or (at your option) the GNU General
+ ## Public license version 3 or any later version approved by the KDE Free
+ ## Qt Foundation. The licenses are as published by the Free Software
+ ## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+ ## included in the packaging of this file. Please review the following
+ ## information to ensure the GNU General Public License requirements will
+ ## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+ ##
+ ## $QT_END_LICENSE$
+ ##
+ ############################################################################
+
+ import argparse
+ import os
+ import sys
+
+ import yaml
+
+ from tools import Namespace, config as Config, state, toolbox, dependency_resolver, repo as Repo
+
+
+ def parse_args(print_help: bool = False) -> Namespace:
+ parser = argparse.ArgumentParser(description="Execute a round of dependency updates by branch.", formatter_class=argparse.RawTextHelpFormatter)
+ parser.add_argument('--sim', dest='simulate', action='store_true',
+ help="Simulate a run of the tool, but don't send any alerts or save\n"
+ "the final state.")
+ parser.add_argument('--reset', dest='reset', action='store_true',
+ help="Forget current update state of [branch], then exit. Requires branch.")
+ parser.add_argument('--pauseOnFail', dest='pause_on_finish_fail', action='store_true',
+ help="If the round finished with failures in blocking repos, do not reset\n"
+ "the round. Hold the current state until rewound or reset.")
+ parser.add_argument('--retryFailed', dest='retry_failed', action='store_true',
+ help="Retries failed updates on top of the branch HEAD for a given module.\n"
+ "Generally only used when a round has failed via use of --pauseOnFail\n"
+ "or when non-blocking modules fail to merge.")
+ parser.add_argument('--reset-stage-count', dest='reset_stage_count', action='store_true',
+ help="Reset all in-progress and retrying repos' stage attempt counters.\n"
+ "Useful intervention in an ongoing round which is about to fail.")
+ parser.add_argument('-b', '--branch', dest='branch', type=str, default="dev",
+ help="Branch to update against.")
+ parser.add_argument('--noState', dest='no_state', action='store_true',
+ help="Perform this update isolated from any saved state for this branch.\n"
+ "Do not save state when completed. Enable this switch to perform\n"
+ "one-off updates of repos independent of a normal round. Will not\n"
+ "interfere with ongoing rounds.",
+ default=False)
+ parser.add_argument('--default-repos', dest='update_default_repos', action='store_true',
+ help="Update all modules in qt5 marked as 'essential',\n"
+ " 'addon', 'deprecated', 'ignore', or 'preview'",
+ default=False)
+ parser.add_argument('-p', '--prefix', dest='repo_prefix', default="qt/",
+ help="Prefer repos with this prefix when choosing the sha to use in\n"
+ "the update. Intended for use against 'qt/tqtc-' repos.")
+ parser.add_argument('--head', dest='use_head', action='store_true',
+ help="Use latest HEAD for all dependencies instead of the latest qt5\n"
+ "dependency map as the starting point.\n"
+ "Implied by --default-repos.")
+ parser.add_argument('-c', '--sweepChanges', dest='sweep_changes', action='store_true',
+ help="Search gerrit for changes with the Submodule Update Bot\n"
+ "(or the current user) added as a reviewer on the change.\n"
+ "Sweep those changes in with this submodule update round.")
+ parser.add_argument('--rewind', dest='rewind_module',
+ help="Rewind the round to the specified module and recalculate\n"
+ "dependencies. Useful to pull in a fix required by leaf modules\n"
+ "without restarting the round.")
+ parser.add_argument('--dropDependency', dest="drop_dependency",
+ help="IMPORTANT: This action is destructive!\n"
+ " FORMAT: dependency[:repo,]\n"
+ " Specify the dependency to drop. If it should be selectively\n"
+ " dropped, follow the dependency with a colon ':', and a\n"
+ " comma-separated list of repos to drop the dependency from.\n"
+ " If a list of repos to drop the dependency from is not supplied,\n"
+ " the dependency will be dropped from ALL repos being processed.")
+ parser.add_argument('-s', '--stage', dest='stage', action='store_true',
+ help="Automatically stage proposed updates if able to self-approve.")
+ parser.add_argument('-q', '--qt5Update', dest='update_supermodule', action='store_true',
+ help="Perform an update to the qt5/qt6 supermodule when all\n"
+ "updates have succeeded")
+ parser.add_argument('--yoctoUpdate', dest='update_yocto_meta', action='store_true',
+ help="Update the yocto/meta-qt6 repo with the shas from this round.")
+ parser.add_argument('-r', '--repos', dest="repos", nargs='*',
+ help="List of repos to update.\n")
+ parser.add_argument('-n', '--nonBlockingRepos', dest="non_blocking_repos", nargs='*',
+ help="List of non-blocking repos to update. These will be included in the\n"
+ "round but will not cause a failure if they fail to integrate unless\n"
+ "another blocking module depends on it.")
+ if print_help:
+ parser.print_help()
+ args = parser.parse_args()
+
+ if args.simulate:
+ print("INFO: Running in simulation mode. No alerts will be sent,"
+ " and state will not be saved!")
+ return args
+
+
+ def clear():
+ """Clear the console screen using the OS built-in methods."""
+ if sys.platform == "win32":
+ os.system('cls')
+ else:
+ os.system('clear')
+
+
+ def main():
+ # Initial setup
+ config = Config._load_config("config.yaml", parse_args())
+ config.datasources.load_datasources(config)
+ config.state_repo = state.check_create_local_repo(config)
+ if config.args.reset:
+ state.clear_state(config)
+ exit()
+ if config.args.update_default_repos:
+ config.args.use_head = True
+ if config.args.rewind_module:
+ config.rewind_module = toolbox.search_for_repo(config, config.args.rewind_module)
+ # Load the state cache
+ config.state_data = state.load_updates_state(config)
+ # Check to see if we should abort as finished-failed
+ if config.state_data.get("pause_on_finish_fail"):
+ if not any([config.args.retry_failed, config.args.rewind_module]):
+ print(
+ "Round is in Failed_finish state and this round was run in Paused On Finish Fail Mode.\n"
+ "To move the round forward, run the script with one of the following --reset,"
+ " --rewind, or --retry_failed")
+ parse_args(print_help=True)
+ exit()
+ # Continue the round and try again.
+ del config.state_data["pause_on_finish_fail"]
+ if config.args.retry_failed:
+ for module in [r for r in config.state_data.values()
+ if r.progress == Repo.PROGRESS.DONE_FAILED_BLOCKING]:
+ toolbox.reset_module_properties(config, module)
+ report_new_round = False
+ if not config.state_data and config.args.update_default_repos:
+ # State data is always empty if the round is fresh.
+ report_new_round = True
+
+ # Collect the list of qt5 modules for our reference.
+ config.qt5_default = toolbox.get_qt5_submodules(config, ["essential", "addon", "deprecated",
+ "preview"])
+
+ # Collect Repo objects for everything in the cache or list of qt5 modules, as necessary.
+ repos = toolbox.get_repos(config)
+ if repos.get(f"{config.args.repo_prefix}qtbase") and report_new_round:
+ qtbase = repos[f"{config.args.repo_prefix}qtbase"]
+ config.teams_connector.send_teams_webhook_basic(
+ repo=qtbase,
+ text=f"INFO: New round started on {qtbase.branch} with"
+ f" {qtbase.id}@{qtbase.original_ref}")
+ # Update the working state with any newly added repos passed to the script.
+ config.state_data = state.update_state_data(config.state_data, repos)
+
+ # Update the progress of all repos in the state since the last run of the tool.
+ for repo in config.state_data.values():
+ repo.progress, repo.proposal.merged_ref, repo.proposal.gerrit_status = \
+ toolbox.get_check_progress(config, repo)
+
+ # Collect necessary data if dropping a dependency from a repo.
+ if config.args.drop_dependency:
+ split = config.args.drop_dependency.split(":")
+ config.drop_dependency = toolbox.search_for_repo(config, split[0])
+ if len(split) > 1:
+ config.drop_dependency_from = \
+ [toolbox.search_for_repo(config, r) for r in split[1].split(",")]
+ else:
+ config.drop_dependency_from = repos
+
+ # Discover dependencies and add any missing repos to the list. We might need to update them too.
+ config.state_data = dependency_resolver.discover_repo_dependencies(config)
+
+ # Mark non-blocking repos as blocking if a blocking repo depends on it.
+ config.state_data = dependency_resolver.cross_check_non_blocking_repos(config)
+
+ # Undo any work done in modules which depend on rewind_module, if set.
+ if config.args.rewind_module:
+ # Set the module to rewind to so that we generate new proposals for
+ # any modules which depend directly or indirectly on it.
+ if config.state_data[config.rewind_module.id].progress < Repo.PROGRESS.DONE:
+ # Rewinding to a module which hasn't merged yet will break the round!
+ print(f"Unable to rewind to a not-yet-updated module. {config.rewind_module.id}"
+ f" is in state: {config.state_data[config.rewind_module.id].progress.name}."
+ f"\nHint: Try rewinding to one if its dependencies:"
+ f" {config.state_data[config.rewind_module.id].dep_list}")
+ else:
+ config.state_data[config.rewind_module.id].proposal.change_id = ""
+ new_sha = toolbox.get_head(config, config.state_data[config.rewind_module.id], True)
+ print(f"\nRewinding round to {config.rewind_module.id} @ {new_sha}\n")
+ config.state_data[config.rewind_module.id].original_ref = new_sha
+ config.state_data[config.rewind_module.id].proposal.merged_ref = new_sha
+ config.state_data[config.rewind_module.id].progress = Repo.PROGRESS.DONE_NO_UPDATE
+ config.teams_connector.send_teams_webhook_basic(
+ repo=config.rewind_module,
+ text=f"INFO: Rewinding '{config.args.branch}' to {new_sha}."
+ f" Modules depending on {config.rewind_module.id} have been reset.")
+ if config.args.update_supermodule and config.state_data.get("qt/qt5") \
+ and not config.rewind_module.id == "yocto/meta-qt6":
+ del config.state_data["qt/qt5"]
+ if config.args.update_yocto_meta and config.state_data.get("yocto/meta-qt6") \
+ and not config.rewind_module.id == "qt/qt5":
+ del config.state_data["yocto/meta-qt6"]
+
+ # bump the progress of repos that have had updates pushed and merged.
+ for repo in config.state_data.values():
+ repo.progress, repo.proposal.merged_ref, repo.proposal.gerrit_status = \
+ toolbox.get_check_progress(config, repo)
+
+ # Retry any modules which are ready but have failed to merge in CI.
+ for repo in [r for r in config.state_data.values() if r.progress == Repo.PROGRESS.RETRY]:
+ if config.args.reset_stage_count:
+ repo = toolbox.reset_stage_count(repo)
+ if repo.retry_count < 3:
+ if repo.retry_count == 1:
+ # Send a warning message if the update has failed to merge twice.
+ print(f"Collecting log snippet for failed integration in {repo.id}...")
+ failed_tests_snip = toolbox.parse_failed_integration_log(config, repo)
+ print(failed_tests_snip)
+ config.teams_connector.send_teams_webhook_module_failed(repo,
+ text_override=f"Dependency update on *{repo.id}*"
+ f" is failing in **{repo.branch}**. Two automatic retries left.",
+ test_failures=failed_tests_snip,
+ pause_links=True)
+ if config.args.stage:
+ repo = toolbox.retry_update(config, repo)
+ else:
+ print(
+ f"WARN: Unable to re-stage {repo.id} because automatic staging is not enabled.\n"
+ f"You must stage {repo.proposal.change_id} manually!")
+ elif repo.is_non_blocking:
+ print(f"Dependency Update to non-blocking repo {repo.id} failed.")
+ repo.progress = Repo.PROGRESS.DONE_FAILED_NON_BLOCKING
+ else:
+ # Clear state and reset, or allow broken updates to fail so others can be updated.
+ # state.clear_state(config) # Hard-disabled reset for now due to long turn-around-time of bugfixes.
+ print(f"Dependency Update to {repo.id} failed.")
+ repo.progress = Repo.PROGRESS.DONE_FAILED_BLOCKING
+ config.teams_connector.send_teams_webhook_module_failed(repo,
+ test_failures=toolbox.parse_failed_integration_log(
+ config, repo))
+
+ # Check and see if we're ready to push a supermodule update if all the blocking repos
+ # Have finished updating successfully.
+ config.state_data = toolbox.do_try_supermodule_updates(config)
+
+ # Finally, we're ready to start resolving dependencies for modules which are PROGRESS.READY
+ for repo in [r for r in config.state_data.values() if r.progress < Repo.PROGRESS.IN_PROGRESS]:
+ print(f"Checking inconsistencies in: {repo.id}")
+ repo.proposal.inconsistent_set = \
+ dependency_resolver.discover_dep_inconsistencies(config, repo)
+ print(f"{repo.id}: {repo.proposal.inconsistent_set}")
+
+ # Generate current_state for later comparison to comprehend if the script took any action
+ current_state, formatted_state = toolbox.state_printer(config)
+ print("\n-=-=-=-=-=-State before pushing updates-=-=-=-=-=-")
+ print(formatted_state)
+ print("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n")
+
+ # Create new dependencies.yaml proposals for all PROGRESS.READY modules.
+ config.state_data = dependency_resolver.recursive_prepare_updates(config)
+
+ for repo in [r for r in config.state_data.values() if r.progress == Repo.PROGRESS.READY]:
+ print(f"Proposed update to {repo.id}:")
+ print("-----------------------------")
+ print(yaml.dump(repo.proposal.proposed_yaml))
+ print("-----------------------------")
+ print()
+
+ # Do the actual gerrit pushes and staging of changes.
+ if not config.args.simulate:
+ for repo in [r for r in config.state_data.values() if
+ r.progress == Repo.PROGRESS.READY and not r.is_supermodule]:
+ repo.proposal.change_id, repo.proposal.change_number \
+ = toolbox.search_existing_change(config, repo, "Update dependencies")
+ repo.proposal = toolbox.push_submodule_update(config, repo)
+ if repo.proposal.change_id:
+ repo.progress = Repo.PROGRESS.IN_PROGRESS
+ elif repo.proposal.merged_ref:
+ repo.progress = Repo.PROGRESS.DONE_NO_UPDATE
+ for repo in [r for r in config.state_data.values() if
+ r.progress == Repo.PROGRESS.IN_PROGRESS]:
+ if config.args.stage and toolbox.stage_update(config, repo):
+ repo.stage_count += 1
+
+ # Check a second time if we need to do a supermodule update, as the above step may
+ # have resulted in a bunch of repos considered PROGRESS.DONE_NO_UPDATE
+ config.state_data = toolbox.do_try_supermodule_updates(config)
+
+ final_state, formatted_state = toolbox.state_printer(config)
+ if final_state != current_state:
+ print("\n-=-=-=-=-=-State after pushing updates-=-=-=-=-=-")
+ print(formatted_state)
+ print("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n")
+ else:
+ print("No updates pushed this round. Nothing else to do this run.")
+
+ # Determine how to exit
+ clear_state = False
+ if not any([r.progress for r in config.state_data.values() if
+ r.progress < Repo.PROGRESS.DONE.value]):
+ if config.args.simulate:
+ print("INFO: Done with this round, but not clearing state because --sim was used.")
+ elif config.args.pause_on_finish_fail and not config.state_data.get("pause_on_finish_fail"):
+ print(
+ "Done with this round: Running in Pause On Finish Fail mode. Not resetting state.")
+ config.state_data["pause_on_finish_fail"] = Repo.Repo(id="pause_on_finish_fail",
+ prefix="",
+ progress=Repo.PROGRESS.IGNORE_IS_META)
+ config.teams_connector.send_teams_webhook_finish_failed(
+ text=f"Update round on {config.args.branch} failed with errors."
+ f" Pausing the round until rewind/reset.", config=config, reset_links=True)
+ else:
+ # Everything was successful! Hooray! The round finished.
+ print("Done with this round! Clearing state.")
+ clear_state = True
+ config.teams_connector.send_teams_webhook_basic(
+ text=f"INFO: Reset/Finished update round on '{config.args.branch}'")
+
+ # Dump the state to disk and save to codereview if available.
+ state.save_updates_state(config, clear_state)
+
+
+ if __name__ == '__main__':
+ main()