Revision control
- # Copyright (C) 2020 The Qt Company Ltd.
- # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
- import copy
- from typing import Union
-
- from tools import toolbox
- from .config import Config
- from .proposal import Proposal
- from .repo import Repo, PROGRESS
-
-
- def recursive_prepare_updates(config) -> dict[str, Repo]:
- """Iterate through the list of repos and prepare updates for all
- in the READY state, or bump the repo to DONE_NO_UPDATE if it has
- no dependencies. Re-perform the operation as many times as needed
- until all eligible repos have had their update created."""
- reload = False
- for repo in config.state_data.values():
- config.state_data[repo.id], trigger_reload = prepare_update(config, repo)
- reload = bool(reload + trigger_reload)
- for repo in config.state_data.values():
- if repo.progress == PROGRESS.READY and not repo.dep_list and not repo.is_supermodule:
- reload = True
- repo.progress = PROGRESS.DONE_NO_UPDATE
- repo.proposal.merged_ref = repo.original_ref
- repo.proposal.proposed_yaml = repo.deps_yaml
- if reload:
- config.state_data = recursive_prepare_updates(config)
- return config.state_data
-
-
- def prepare_update(config: Config, repo: Repo) -> tuple[Repo, bool]:
- """Bump progress of a repo if it's dependencies are met,
- then create a proposal if it doesn't already exist."""
-
- repo.progress, progress_changed, repo.failed_dependencies = determine_ready(config, repo)
- if repo.is_supermodule:
- return repo, False
- repo.proposal = retrieve_or_generate_proposal(config, repo)
- reload = False
- if progress_changed and repo.progress >= PROGRESS.DONE:
- reload = True
- elif not repo.proposal and repo.progress < PROGRESS.WAIT_DEPENDENCY:
- print(f"moving {repo.id} to DONE_NO_UPDATE")
- repo.progress = PROGRESS.DONE_NO_UPDATE
- repo.proposal.merged_ref = repo.original_ref
- repo.proposal.proposed_yaml = repo.deps_yaml
- reload = True
- return repo, reload
-
-
- def retrieve_or_generate_proposal(config: Config, repo) -> Proposal:
- """Return the proposed YAML if it exists and should not be updated,
- otherwise, generate a new one with the latest shas."""
- if repo.progress in [PROGRESS.DONE_FAILED_DEPENDENCY, PROGRESS.DONE_FAILED_NON_BLOCKING,
- PROGRESS.DONE_FAILED_BLOCKING, PROGRESS.DONE, PROGRESS.DONE_NO_UPDATE,
- PROGRESS.WAIT_DEPENDENCY, PROGRESS.WAIT_INCONSISTENT,
- PROGRESS.IN_PROGRESS]:
- return repo.proposal
- else:
- # Create a new proposal for failed updates if the user has specified
- # --retryFailed in the arguments.
- if repo.progress == PROGRESS.RETRY and not config.args.retry_failed:
- return repo.proposal
- print(f"Creating new proposal for {repo.id}: {repo.progress.name}")
- proposal = copy.deepcopy(repo.deps_yaml)
- for dep in repo.deps_yaml.get("dependencies"):
- prefix, dep_name = toolbox.strip_prefix(dep)
- full_name = [n for n in repo.dep_list if dep_name in n].pop()
- proposal["dependencies"][dep]["ref"] = toolbox.get_head(config, full_name)[0]
- if proposal == repo.deps_yaml:
- print(f"{repo.id} dependencies are already up-to-date")
- else:
- repo.proposal.proposed_yaml = proposal
- return repo.proposal
-
-
- def check_subtree(config, source: Repo,
- source_ref: str,
- target: Repo) -> tuple[str, tuple[str, str]]:
- """Compare a sha between two repos' dependencies.yaml references for the same dependency.
- Recurse for each dependency which is not the same as the source.
-
- :returns: the id of a target repo which has a mismatching sha to the source_ref"""
- deps = target.deps_yaml.get("dependencies") \
- if target.progress < PROGRESS.DONE \
- or target.progress == Repo.progress.DONE_FAILED_DEPENDENCY \
- or target.progress in (Repo.progress.DONE_FAILED_BLOCKING,
- Repo.progress.DONE_FAILED_NON_BLOCKING) \
- else target.proposal.proposed_yaml.get("dependencies")
- for dependency in deps.keys():
- if source.name in dependency:
- if not source_ref == deps[dependency]["ref"]:
- if source.proposal.merged_ref == deps[dependency]["ref"]:
- continue
- else:
- print(f"source {source.name}:{source_ref[:10]} is not the same as {source.name}"
- f" in {target.name}->{dependency}:{deps[dependency]['ref']}") # Verbose!
- return source.id, (target.id, deps[dependency]["ref"])
- else:
- clean_name = toolbox.strip_prefix(dependency)[1]
- new_target = config.state_data[toolbox.search_for_repo(config, clean_name).id]
- return check_subtree(config, source, source_ref, new_target)
- return tuple()
-
-
- def discover_dep_inconsistencies(config: Config, repo: Repo) \
- -> dict[str, set[str]]:
- """Traverse the dependency tree of a repo, finding mismatching shas among dependency
- refrences. This allows us to determine the lowest-level module that must be updated
- in order to begin or continue a round."""
- mismatches = dict()
- if not repo.deps_yaml.get("dependencies"):
- return {}
- for top_dep in repo.deps_yaml["dependencies"].keys():
- top_dep_repo = toolbox.search_for_repo(config, top_dep)
- for other_dep in repo.deps_yaml["dependencies"].keys():
- other_dep_repo = toolbox.search_for_repo(config, other_dep)
- top_dep_ref = repo.deps_yaml["dependencies"][top_dep]["ref"]
- temp = check_subtree(config, top_dep_repo, top_dep_ref, other_dep_repo)
- if temp:
- if not temp[0] in mismatches.keys():
- mismatches[temp[0]] = set()
- mismatches[temp[0]].add(temp[1])
- return mismatches
-
-
- def discover_repo_dependencies(config: Config,
- repos_override: list[Repo] = None) -> dict[str, Repo]:
- """Traverse the dependency tree for a repo and add any repos found
- to the list of repos to update if it was not already specified
- or found to be part of qt5 default modules.
- Compile the list of dependencies for a given repo, direct and indirect."""
- for repo in repos_override or copy.deepcopy(list(config.state_data.values())):
- dep_list = set()
- if repo.id in [repo.prefix + "qt5", "yocto/meta-qt6"]:
- continue
- if repo.progress >= PROGRESS.IN_PROGRESS and not config.rewind_module:
- # If a module is done, or had no update, we don't care about its dependencies.
- # This means that if we're discovering against qt5.git submodule shas,
- # we won't traverse the tree at all since we don't care about what a
- # direct dependency requires.
- continue
- if not repo.deps_yaml:
- repo.deps_yaml, repo.branch = toolbox.get_dependencies_yaml(config, repo)
- for dep in repo.deps_yaml.get('dependencies'):
- # Recurse through the tree until no more dependencies are found.
- relative_prefix, bare_dep = toolbox.strip_prefix(dep)
- key, dep_repo = toolbox.get_repos(config, [bare_dep], None).popitem()
- dep_list.add(dep_repo.id)
- config.state_data.update({key: dep_repo})
- # Retrieve the complete list of dependencies for this dependency.
- sub_deps = discover_repo_dependencies(config, [dep_repo])[dep_repo.id].dep_list
- # Add these dependencies to the master list for the repo we were first looking at.
- dep_list.update(sub_deps)
- repo.dep_list = list(dep_list)
- # Update this repo in our master list of repos.
- config.state_data.update({repo.id: repo})
- # Cross-check that we didn't miss anything
- config.state_data = discover_missing_dependencies(config, repo)
- if config.rewind_module and config.rewind_module.id in config.state_data[repo.id].dep_list:
- # If the module depends on the module we needed to rewind
- # to, either directly or indirectly, reset the state
- # and treat it as though it hasn't been updated.
- config.state_data[repo.id] = \
- toolbox.reset_module_properties(config, config.state_data[repo.id])
- return config.state_data
-
-
- def cross_check_non_blocking_repos(config: Config) -> dict[str, Repo]:
- """Examine dependencies of all blocking repos. Convert a repo from non-blocking status
- to blocking if it is a dependency of a blocking repo to be updated."""
- for repo in config.state_data.values():
- if not repo.is_non_blocking:
- if repo.dep_list:
- for dep in repo.dep_list:
- if config.state_data.get(dep) and config.state_data.get(dep).is_non_blocking:
- config.state_data[dep].is_non_blocking = False
- return config.state_data
-
-
- def discover_missing_dependencies(config: Config, repo: Repo) -> dict[str, Repo]:
- """Given a repo's dependency list, check for missing repos in the
- state_data and add them"""
- for dep in repo.dep_list:
- if dep not in config.state_data.keys():
- # Initialize the missing repo with the minimum data needed.
- temp_repo = toolbox.get_repos(config, repos_override=[dep], non_blocking_override=None)[dep]
- config.state_data[temp_repo.id] = temp_repo
- config.state_data = discover_repo_dependencies(config)
- return config.state_data
-
-
- def determine_ready(config: Config, repo: Repo) -> tuple[PROGRESS, bool, Union[list, None]]:
- """Check to see if a repo is waiting on another, or if all
- dependency conflicts have been resolved and/or updated."""
- worst_state = PROGRESS.READY
- # Keep a list of failed dependencies so we can explain
- # why a repo is being marked as failed without
- # actually attempting an update.
- failed_repos = set()
-
- def is_worse(state):
- nonlocal worst_state
- if state > worst_state:
- worst_state = state
-
- if repo.proposal.inconsistent_set:
- is_worse(PROGRESS.WAIT_INCONSISTENT)
- if repo.progress < PROGRESS.IN_PROGRESS \
- or repo.progress == PROGRESS.DONE_FAILED_DEPENDENCY:
- for dependency in repo.dep_list:
- try:
- dep_repo = config.state_data[dependency]
- except Exception as e:
- print(f"{repo.id} requested nonexistent dependency: {dependency}")
- if dependency in config.state_data.keys():
- # Recurse and update the progress in the case that we're trying
- # to rewind with many previously failed dependencies.
- dep_repo.progress, _, dep_repo.failed_dependencies \
- = determine_ready(config, dep_repo)
-
- if dep_repo.progress < PROGRESS.DONE:
- is_worse(PROGRESS.WAIT_DEPENDENCY)
- elif dep_repo.progress == PROGRESS.DONE_FAILED_NON_BLOCKING:
- print(f"WARN: {repo.id} dependency {dependency} is a non-blocking module which"
- f" failed. Marking {repo.id} as Failed due to dependency.")
- failed_repos.add(dependency)
- is_worse(PROGRESS.DONE_FAILED_DEPENDENCY)
- elif dep_repo.progress == PROGRESS.DONE_FAILED_BLOCKING:
- print(f"WARN: {repo.id} dependency {dependency} is a blocking module which"
- f" failed. Marking {repo.id} as Failed due to dependency.")
- is_worse(PROGRESS.DONE_FAILED_DEPENDENCY)
- failed_repos.add(dependency)
- return worst_state, repo.progress != worst_state, list(failed_repos)
- else:
- return repo.progress, False, None