|
@@ -14,16 +14,10 @@
|
|
|
# more details.
|
|
|
#
|
|
|
import argparse
|
|
|
-import glob
|
|
|
-import json
|
|
|
import logging
|
|
|
-import math
|
|
|
import os
|
|
|
import re
|
|
|
import sys
|
|
|
-from collections import namedtuple, OrderedDict
|
|
|
-from datetime import datetime, timedelta, tzinfo
|
|
|
-from operator import attrgetter
|
|
|
|
|
|
# Import oe and bitbake libs
|
|
|
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
|
@@ -34,128 +28,13 @@ scriptpath.add_oe_lib_path()
|
|
|
|
|
|
from oeqa.utils.git import GitRepo, GitError
|
|
|
from oeqa.utils.metadata import metadata_from_bb
|
|
|
-
|
|
|
+import oeqa.utils.gitarchive as gitarchive
|
|
|
|
|
|
# Setup logging
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
|
|
log = logging.getLogger()
|
|
|
|
|
|
|
|
|
-class ArchiveError(Exception):
|
|
|
- """Internal error handling of this script"""
|
|
|
-
|
|
|
-
|
|
|
-def format_str(string, fields):
|
|
|
- """Format string using the given fields (dict)"""
|
|
|
- try:
|
|
|
- return string.format(**fields)
|
|
|
- except KeyError as err:
|
|
|
- raise ArchiveError("Unable to expand string '{}': unknown field {} "
|
|
|
- "(valid fields are: {})".format(
|
|
|
- string, err, ', '.join(sorted(fields.keys()))))
|
|
|
-
|
|
|
-
|
|
|
-def init_git_repo(path, no_create, bare):
|
|
|
- """Initialize local Git repository"""
|
|
|
- path = os.path.abspath(path)
|
|
|
- if os.path.isfile(path):
|
|
|
- raise ArchiveError("Invalid Git repo at {}: path exists but is not a "
|
|
|
- "directory".format(path))
|
|
|
- if not os.path.isdir(path) or not os.listdir(path):
|
|
|
- if no_create:
|
|
|
- raise ArchiveError("No git repo at {}, refusing to create "
|
|
|
- "one".format(path))
|
|
|
- if not os.path.isdir(path):
|
|
|
- try:
|
|
|
- os.mkdir(path)
|
|
|
- except (FileNotFoundError, PermissionError) as err:
|
|
|
- raise ArchiveError("Failed to mkdir {}: {}".format(path, err))
|
|
|
- if not os.listdir(path):
|
|
|
- log.info("Initializing a new Git repo at %s", path)
|
|
|
- repo = GitRepo.init(path, bare)
|
|
|
- try:
|
|
|
- repo = GitRepo(path, is_topdir=True)
|
|
|
- except GitError:
|
|
|
- raise ArchiveError("Non-empty directory that is not a Git repository "
|
|
|
- "at {}\nPlease specify an existing Git repository, "
|
|
|
- "an empty directory or a non-existing directory "
|
|
|
- "path.".format(path))
|
|
|
- return repo
|
|
|
-
|
|
|
-
|
|
|
-def git_commit_data(repo, data_dir, branch, message, exclude, notes):
|
|
|
- """Commit data into a Git repository"""
|
|
|
- log.info("Committing data into to branch %s", branch)
|
|
|
- tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive')
|
|
|
- try:
|
|
|
- # Create new tree object from the data
|
|
|
- env_update = {'GIT_INDEX_FILE': tmp_index,
|
|
|
- 'GIT_WORK_TREE': os.path.abspath(data_dir)}
|
|
|
- repo.run_cmd('add .', env_update)
|
|
|
-
|
|
|
- # Remove files that are excluded
|
|
|
- if exclude:
|
|
|
- repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update)
|
|
|
-
|
|
|
- tree = repo.run_cmd('write-tree', env_update)
|
|
|
-
|
|
|
- # Create new commit object from the tree
|
|
|
- parent = repo.rev_parse(branch)
|
|
|
- git_cmd = ['commit-tree', tree, '-m', message]
|
|
|
- if parent:
|
|
|
- git_cmd += ['-p', parent]
|
|
|
- commit = repo.run_cmd(git_cmd, env_update)
|
|
|
-
|
|
|
- # Create git notes
|
|
|
- for ref, filename in notes:
|
|
|
- ref = ref.format(branch_name=branch)
|
|
|
- repo.run_cmd(['notes', '--ref', ref, 'add',
|
|
|
- '-F', os.path.abspath(filename), commit])
|
|
|
-
|
|
|
- # Update branch head
|
|
|
- git_cmd = ['update-ref', 'refs/heads/' + branch, commit]
|
|
|
- if parent:
|
|
|
- git_cmd.append(parent)
|
|
|
- repo.run_cmd(git_cmd)
|
|
|
-
|
|
|
- # Update current HEAD, if we're on branch 'branch'
|
|
|
- if not repo.bare and repo.get_current_branch() == branch:
|
|
|
- log.info("Updating %s HEAD to latest commit", repo.top_dir)
|
|
|
- repo.run_cmd('reset --hard')
|
|
|
-
|
|
|
- return commit
|
|
|
- finally:
|
|
|
- if os.path.exists(tmp_index):
|
|
|
- os.unlink(tmp_index)
|
|
|
-
|
|
|
-
|
|
|
-def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
|
|
|
- keywords):
|
|
|
- """Generate tag name and message, with support for running id number"""
|
|
|
- keyws = keywords.copy()
|
|
|
- # Tag number is handled specially: if not defined, we autoincrement it
|
|
|
- if 'tag_number' not in keyws:
|
|
|
- # Fill in all other fields than 'tag_number'
|
|
|
- keyws['tag_number'] = '{tag_number}'
|
|
|
- tag_re = format_str(name_pattern, keyws)
|
|
|
- # Replace parentheses for proper regex matching
|
|
|
- tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$'
|
|
|
- # Inject regex group pattern for 'tag_number'
|
|
|
- tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
|
|
|
-
|
|
|
- keyws['tag_number'] = 0
|
|
|
- for existing_tag in repo.run_cmd('tag').splitlines():
|
|
|
- match = re.match(tag_re, existing_tag)
|
|
|
-
|
|
|
- if match and int(match.group('tag_number')) >= keyws['tag_number']:
|
|
|
- keyws['tag_number'] = int(match.group('tag_number')) + 1
|
|
|
-
|
|
|
- tag_name = format_str(name_pattern, keyws)
|
|
|
- msg_subj= format_str(msg_subj_pattern.strip(), keyws)
|
|
|
- msg_body = format_str(msg_body_pattern, keyws)
|
|
|
- return tag_name, msg_subj + '\n\n' + msg_body
|
|
|
-
|
|
|
-
|
|
|
def parse_args(argv):
|
|
|
"""Parse command line arguments"""
|
|
|
parser = argparse.ArgumentParser(
|
|
@@ -217,17 +96,11 @@ def get_nested(d, list_of_keys):
|
|
|
return ""
|
|
|
|
|
|
def main(argv=None):
|
|
|
- """Script entry point"""
|
|
|
args = parse_args(argv)
|
|
|
if args.debug:
|
|
|
log.setLevel(logging.DEBUG)
|
|
|
|
|
|
try:
|
|
|
- if not os.path.isdir(args.data_dir):
|
|
|
- raise ArchiveError("Not a directory: {}".format(args.data_dir))
|
|
|
-
|
|
|
- data_repo = init_git_repo(args.git_dir, args.no_create, args.bare)
|
|
|
-
|
|
|
# Get keywords to be used in tag and branch names and messages
|
|
|
metadata = metadata_from_bb()
|
|
|
keywords = {'hostname': get_nested(metadata, ['hostname']),
|
|
@@ -236,39 +109,12 @@ def main(argv=None):
|
|
|
'commit_count': get_nested(metadata, ['layers', 'meta', 'commit_count']),
|
|
|
'machine': get_nested(metadata, ['config', 'MACHINE'])}
|
|
|
|
|
|
- # Expand strings early in order to avoid getting into inconsistent
|
|
|
- # state (e.g. no tag even if data was committed)
|
|
|
- commit_msg = format_str(args.commit_msg_subject.strip(), keywords)
|
|
|
- commit_msg += '\n\n' + format_str(args.commit_msg_body, keywords)
|
|
|
- branch_name = format_str(args.branch_name, keywords)
|
|
|
- tag_name = None
|
|
|
- if not args.no_tag and args.tag_name:
|
|
|
- tag_name, tag_msg = expand_tag_strings(data_repo, args.tag_name,
|
|
|
- args.tag_msg_subject,
|
|
|
- args.tag_msg_body, keywords)
|
|
|
-
|
|
|
- # Commit data
|
|
|
- commit = git_commit_data(data_repo, args.data_dir, branch_name,
|
|
|
- commit_msg, args.exclude, args.notes)
|
|
|
-
|
|
|
- # Create tag
|
|
|
- if tag_name:
|
|
|
- log.info("Creating tag %s", tag_name)
|
|
|
- data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit])
|
|
|
-
|
|
|
- # Push data to remote
|
|
|
- if args.push:
|
|
|
- cmd = ['push', '--tags']
|
|
|
- # If no remote is given we push with the default settings from
|
|
|
- # gitconfig
|
|
|
- if args.push is not True:
|
|
|
- notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name)
|
|
|
- for ref, _ in args.notes]
|
|
|
- cmd.extend([args.push, branch_name] + notes_refs)
|
|
|
- log.info("Pushing data to remote")
|
|
|
- data_repo.run_cmd(cmd)
|
|
|
+ gitarchive.gitarchive(args.data_dir, args.git_dir, args.no_create, args.bare,
|
|
|
+ args.commit_msg_subject.strip(), args.commit_msg_body, args.branch_name,
|
|
|
+ args.no_tag, args.tag_name, args.tag_msg_subject, args.tag_msg_body,
|
|
|
+ args.exclude, args.notes, args.push, keywords, log)
|
|
|
|
|
|
- except ArchiveError as err:
|
|
|
+ except gitarchive.ArchiveError as err:
|
|
|
log.error(str(err))
|
|
|
return 1
|
|
|
|