kernel-package.py 14.7 KB
Newer Older
1
#!/usr/bin/env python2
Igor Gnatenko's avatar
Igor Gnatenko committed
2

3
# Copyright 2014 Igor Gnatenko
Igor Gnatenko's avatar
Igor Gnatenko committed
4 5
# Author(s): Igor Gnatenko <i.gnatenko.brain AT gmail DOT com>
#
Igor Gnatenko's avatar
Igor Gnatenko committed
6 7 8 9
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
Igor Gnatenko's avatar
Igor Gnatenko committed
10 11 12 13 14
# See http://www.gnu.org/copyleft/gpl.html for the full text of the license.

import os
import sys
import argparse
Igor Gnatenko's avatar
Igor Gnatenko committed
15 16
import urlgrabber
import urlgrabber.progress
Igor Gnatenko's avatar
Igor Gnatenko committed
17 18
import git
import re
Igor Gnatenko's avatar
Igor Gnatenko committed
19 20
import subprocess
import stat
21
import glob
22
import shutil
23
import signal
Igor Gnatenko's avatar
Igor Gnatenko committed
24
from HTMLParser import HTMLParser
Igor Gnatenko's avatar
Igor Gnatenko committed
25

Igor Gnatenko's avatar
Igor Gnatenko committed
26
WORK_DIR = os.getcwd()
Igor Gnatenko's avatar
Igor Gnatenko committed
27 28 29 30 31
srcs = []


class ConfigfilesHTMLParser(HTMLParser):
    def handle_data(self, data):
32
        if data.startswith("kernel-") and data.endswith(".config") and data != "kernel-local":
Igor Gnatenko's avatar
Igor Gnatenko committed
33
            srcs.append(data)
Igor Gnatenko's avatar
Igor Gnatenko committed
34

Igor Gnatenko's avatar
Igor Gnatenko committed
35

Igor Gnatenko's avatar
Igor Gnatenko committed
36
class Options:
Igor Gnatenko's avatar
Igor Gnatenko committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
    def __init__(self, work_dir):
        signal.signal(signal.SIGINT, self.handler_clean)
        try:
            self.repo = git.Repo(work_dir)
        except git.exc.InvalidGitRepositoryError:
            print("Wtf? This folder not contains valid git repository!")
            sys.exit(1)
        assert not self.repo.bare
        self.repo.config_reader()
        self.name = "kernel"
        self.hcommit = self.repo.head.commit
        self.sha = self.hcommit.hexsha
        try:
            self.author = self.hcommit.author
            self.summary = self.hcommit.summary
        except LookupError:
            print("Please fix https://github.com/gitpython-developers/GitPython/pull/57 before!")
            sys.exit(1)
55
        self.git_url = "http://gitlab.incom.co/webgeek1234/kernel"
Igor Gnatenko's avatar
Igor Gnatenko committed
56 57 58 59 60 61 62 63 64
        self.prefix = None
        self.format = "tar.gz"
        self.patch = None
        self.directory = "sources"
        self.ver = [None, None, None, None, None]
        self.released = False
        self.released_candidate = False
        self.get_kernel_info()
        self.prefix = "linux-{}.{}".format(self.ver[0], self.ver[1] if self.released else (int(self.ver[1]) - 1))
65 66 67 68
        self.sources = ["check_configs.awk", "cpupower.config", "cpupower.service", "generate_all_configs.sh",
                        "generate_debug_configs.sh", "Makefile", "merge.pl", "mod-extra.list",
                        "mod-extra.sh", "mod-sign.sh", "remove-binary-diff.pl", "x509.genkey"]
        self.filters = ["filter-modules.sh", "filter-s390x.sh", "filter-ppc64le.sh",
69 70
                        "filter-ppc64.sh", "filter-aarch64.sh", "filter-i686.sh", "filter-armv7hl.sh",
                        "filter-x86_64.sh"]
Igor Gnatenko's avatar
Igor Gnatenko committed
71
        try:
72
            with open("{}/kernel-local".format(self.directory), "r"):
Igor Gnatenko's avatar
Igor Gnatenko committed
73 74 75
                pass
        except IOError, e:
            if e.errno == 2:
76
                self.sources.append("kernel-local")
Igor Gnatenko's avatar
Igor Gnatenko committed
77
        self._get_configs()
78 79
        self.execute = ["check_configs.awk", "generate_all_configs.sh", "generate_debug_configs.sh", "merge.pl",
                        "mod-extra.sh", "mod-sign.sh", "remove-binary-diff.pl"]
Igor Gnatenko's avatar
Igor Gnatenko committed
80

Igor Gnatenko's avatar
Igor Gnatenko committed
81
    def _get_configs(self):
82
        tree = urlgrabber.urlread("{}/tree/master/".format(self.git_url))
Igor Gnatenko's avatar
Igor Gnatenko committed
83 84 85 86 87
        parser = ConfigfilesHTMLParser()
        parser.feed(tree)
        for src in srcs:
            self.sources.append(src)

Igor Gnatenko's avatar
Igor Gnatenko committed
88 89 90
    def handler_clean(self, signum, frame):
        self.clean_tree(True)
        sys.exit(0)
91

Igor Gnatenko's avatar
Igor Gnatenko committed
92 93 94
    def handler_checkout_clean(self, signum, frame):
        self.repo.git.checkout(self.sha)
        self.handler_clean()
Igor Gnatenko's avatar
Igor Gnatenko committed
95

Igor Gnatenko's avatar
Igor Gnatenko committed
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
    def get_kernel_info(self):
        lines = []
        with open("Makefile", "r") as f:
            lines = [f.next() for x in xrange(5)]
        i = 0
        for line in lines:
            self.ver[i] = re.sub(r"^.* = (.*)\n$", r"\1", line)
            i += 1
        if "=" in self.ver[3]:
            self.ver[3] = None
            self.released = True
            if re.search("^Linus Torvalds$", str(self.author)) and \
               re.search("^Linux {}.{}$".format(self.ver[0], self.ver[1]), self.summary):
                self.released_candidate = True
            else:
                if re.search("^Linus Torvalds$", str(self.author)) and \
                   re.search("^Linux {}.{}{}$".format(self.ver[0], self.ver[1], self.ver[3]), self.summary):
113
                    self.released = False
Igor Gnatenko's avatar
Igor Gnatenko committed
114
                    self.released_candidate = True
Igor Gnatenko's avatar
Igor Gnatenko committed
115

Igor Gnatenko's avatar
Igor Gnatenko committed
116 117 118 119 120 121
    def print_info(self):
        print("Version: {}.{}".format(self.ver[0], self.ver[1]) +
              ("{}".format(self.ver[3]) if not self.released else "") +
              ("+" if not self.released_candidate else ""))
        print("Codename: {}").format(self.ver[4])
        print("Commit:\n  Author: {}\n  Summary: {}").format(self.author, self.summary)
122

Igor Gnatenko's avatar
Igor Gnatenko committed
123 124 125 126 127
    def set_execute(self):
        for source in self.execute:
            src = "{}/{}".format(self.directory, source)
            st = os.stat(src)
            os.chmod(src, st.st_mode | stat.S_IEXEC)
Igor Gnatenko's avatar
Igor Gnatenko committed
128

Igor Gnatenko's avatar
Igor Gnatenko committed
129 130
    def download_file(self, file_name):
        pg = urlgrabber.progress.TextMeter()
131
        urlgrabber.urlgrab("{}/raw/master/{}".format(self.git_url, file_name),
Igor Gnatenko's avatar
Igor Gnatenko committed
132
                           "sources/{}".format(file_name), progress_obj=pg)
Igor Gnatenko's avatar
Igor Gnatenko committed
133

Igor Gnatenko's avatar
Igor Gnatenko committed
134 135 136
    def download_sources(self):
        for source in self.sources:
            self.download_file(source)
137 138 139 140 141
        for source in self.filters:
            self.download_file(source)
            src = "{}/{}".format(self.directory, source)
            st = os.stat(src)
            os.chmod(src, st.st_mode | stat.S_IEXEC)
Igor Gnatenko's avatar
Igor Gnatenko committed
142

Igor Gnatenko's avatar
Igor Gnatenko committed
143 144
    def download_spec(self):
        self.download_file("{}.spec".format(self.name))
Igor Gnatenko's avatar
Igor Gnatenko committed
145

Igor Gnatenko's avatar
Igor Gnatenko committed
146 147 148 149
    def download_files(self):
        self.download_spec()
        self.download_sources()
        self.set_execute()
Igor Gnatenko's avatar
Igor Gnatenko committed
150

Igor Gnatenko's avatar
Igor Gnatenko committed
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
    def make_patch(self):
        if not self.released:
            self.patchfile = "{}/patch-{}.{}{}".format(self.directory, self.ver[0], self.ver[1], self.ver[3])
            patch = open(self.patchfile, "w")
            p = subprocess.Popen("git diff v{}.{} v{}.{}{}".format(self.ver[0], (int(self.ver[1]) - 1), self.ver[0], self.ver[1], self.ver[3]),
                                 shell=True, universal_newlines=True, stdout=patch)
            p.wait()
            patch.flush()
            patch.close()
            subprocess.call(["xz", "-z", self.patchfile])
            if not self.released_candidate:
                self.patchfile = "{}/patch-{}.{}{}-git999".format(self.directory, self.ver[0], self.ver[1], self.ver[3])
                patch = open(self.patchfile, "w")
                p = subprocess.Popen("git diff v{}.{}{} {}".format(self.ver[0], self.ver[1], self.ver[3], self.sha),
                                     shell=True, universal_newlines=True, stdout=patch)
                p.wait()
                patch.flush()
                patch.close()
                subprocess.call(["xz", "-z", self.patchfile])
        elif not self.released_candidate:
            self.patchfile = "{}/patch-{}.{}-git999".format(self.directory, self.ver[0], self.ver[1])
            patch = open(self.patchfile, "w")
            p = subprocess.Popen("git diff v{}.{} {}".format(self.ver[0], self.ver[1], self.sha),
                                 shell=True, universal_newlines=True, stdout=patch)
            p.wait()
            patch.flush()
            patch.close()
            subprocess.call(["xz", "-z", self.patchfile])
179

Igor Gnatenko's avatar
Igor Gnatenko committed
180 181 182 183 184 185 186 187 188 189
    def archive(self):
        signal.signal(signal.SIGINT, self.handler_checkout_clean)
        if not self.released:
            self.repo.git.checkout("v{}.{}".format(self.ver[0], (int(self.ver[1]) - 1)))
        f = open("{}/{}.{}".format(self.directory, self.prefix, self.format), "w")
        self.repo.archive(f, prefix="{}/".format(self.prefix), format=self.format)
        f.close()
        if not self.released:
            self.repo.git.checkout(self.sha)
        signal.signal(signal.SIGINT, self.handler_clean)
Igor Gnatenko's avatar
Igor Gnatenko committed
190

Igor Gnatenko's avatar
Igor Gnatenko committed
191 192 193 194 195 196 197 198 199 200
    def parse_spec(self, args):
        lines = []
        with open("{}/{}.spec".format(self.directory, self.name), "r") as f:
            lines = f.readlines()
        first = True
        patches = glob.glob("{}/*.patch".format(self.directory))
        patches.sort()
        i = 0
        while i < len(patches):
            patches[i] = re.sub("{}/".format(self.directory), "", patches[i])
Igor Gnatenko's avatar
Igor Gnatenko committed
201
            i += 1
Igor Gnatenko's avatar
Igor Gnatenko committed
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
        i = 0
        while i < len(lines):
            if re.search("^%changelog", lines[i]):
                try:
                    while True:
                        del lines[i]
                except IndexError:
                    pass
            elif re.search("^%global released_kernel [01]", lines[i]):
                lines[i] = re.sub(r"[01]", "1" if self.released else "0", lines[i])
                i += 1
            elif re.search("^# % define buildid .local", lines[i]):
                lines[i] = re.sub("# % ", "%", lines[i])
                if args.buildid:
                    lines[i] = re.sub("local", "{}.{}".format(self.sha[:8], args.buildid), lines[i])
                else:
                    lines[i] = re.sub("local", "{}".format(self.sha[:8]), lines[i])
                i += 1
            elif re.search("^%define base_sublevel [0-9]+", lines[i]):
                lines[i] = re.sub(r"[0-9]+", self.ver[1] if self.released else (str(int(self.ver[1]) - 1)), lines[i])
                i += 1
            elif re.search("^%define stable_update [0-9]+", lines[i]):
                lines[i] = re.sub(r"[0-9]+", self.ver[2], lines[i])
                i += 1
            elif re.search("^%define rcrev [0-9]+", lines[i]):
                lines[i] = re.sub(r"[0-9]+", re.sub(r"[^0-9]", "", self.ver[3]) if not self.released else "0", lines[i])
                i += 1
            elif re.search("^%define gitrev [0-9]+", lines[i]):
                lines[i] = re.sub(r"[0-9]+", "999" if not self.released_candidate else "0", lines[i])
                i += 1
            elif re.search("^%global baserelease [0-9]+", lines[i]):
                lines[i] = re.sub(r"[0-9]+", "999" if self.released else "1", lines[i])
                i += 1
            elif re.search("^%define debugbuildsenabled [01]", lines[i]):
                lines[i] = re.sub(r"[01]", "1" if args.separate_debug else "0", lines[i])
                i += 1
            elif re.search("^%define rawhide_skip_docs [01]", lines[i]):
                lines[i] = re.sub(r"[01]", "1", lines[i])
                i += 1
            elif re.search("^%define with_vanilla ", lines[i]):
                lines[i] = re.sub(r"[01]}(.*) [01]", r"1}\1 0", lines[i])
                i += 1
            elif re.search("^%define with_debuginfo ", lines[i]):
                lines[i] = re.sub(r"[01]}(.*) [01]", r"1}\1 0", lines[i])
                i += 1
            elif re.search("^%define with_perf ", lines[i]):
                lines[i] = re.sub(r"[01]}(.*) [01]", r"1}\1 0", lines[i])
                i += 1
            elif re.search("^%define listnewconfig_fail [01]", lines[i]) and not args.chk_config:
                lines[i] = re.sub(r"[01]", "0", lines[i])
                i += 1
            elif re.search("^Source0: ", lines[i]):
                lines[i] = re.sub(r" .*$", " {}.{}".format(self.prefix, self.format), lines[i])
                i += 1
            elif re.search("^Source[0-9]+: perf-man.*.tar.gz", lines[i]):
                lines[i] = re.sub(r"^", "#", lines[i])
                i += 1
            elif re.search("^%if !%{nopatches}", lines[i]) and args.patches:
                i += 1
                if first:
                    j = 100
                    for patch in patches:
                        lines.insert(i, "Patch{}: {}\n".format(str(j), patch))
                        j += 1
                        i += 1
                    first = False
                else:
                    for patch in patches:
                        lines.insert(i, "ApplyPatch {}\n".format(patch))
                        i += 1
            elif re.search("^(Patch[0-9]+:|Apply(Optional|)Patch) ", lines[i]) and \
                (re.search("^Patch00: patch-3.%{upstream_sublevel}-rc%{rcrev}.xz", lines[i]) or
                 re.search("^Patch01: patch-3.%{upstream_sublevel}-rc%{rcrev}-git%{gitrev}.xz", lines[i]) or
                 re.search("^Patch00: patch-3.%{base_sublevel}-git%{gitrev}.xz", lines[i])) is None:
                lines[i] = re.sub(r"^", "#", lines[i])
                i += 1
            else:
                i += 1
            f = open("{}/{}.spec".format(self.directory, self.name), "w")
            for line in lines:
                f.write(line)
            f.close()

    def make_srpm(self):
Igor Gnatenko's avatar
Igor Gnatenko committed
286 287 288 289
        subprocess.call(["rpmbuild", "-bs", "{}/{}.spec".format(self.directory, self.name),
                         "-D", "_specdir {}".format(self.directory),
                         "-D", "_sourcedir {}".format(self.directory),
                         "-D", "_srcrpmdir {}".format(self.directory)])
Igor Gnatenko's avatar
Igor Gnatenko committed
290

Igor Gnatenko's avatar
Igor Gnatenko committed
291 292 293 294 295 296 297 298 299 300 301 302 303
    def clean_tree(self, first_clean):
        try:
            os.stat(self.directory)
            if not os.access(self.directory, os.W_OK):
                print("Wtf? I don't have access to '{}/' directory!").format(self.directory)
                sys.exit(1)
        except OSError as e:
            if e.errno == 2:
                os.makedirs(self.directory)
        clean = glob.glob("{}/*".format(self.directory))
        i = 0
        while i < len(clean):
            if re.search(".patch$", clean[i]) or \
304
               re.search("kernel-local$", clean[i]):
Igor Gnatenko's avatar
Igor Gnatenko committed
305 306 307 308 309 310 311 312 313 314 315
                del clean[i]
            elif re.search(".src.rpm$", clean[i]) and not first_clean:
                del clean[i]
            else:
                i += 1
        for to_clean in clean:
            try:
                os.remove(to_clean)
            except OSError as e:
                if e.errno == 21 or e.errno == 39:
                    shutil.rmtree(to_clean)
Igor Gnatenko's avatar
Igor Gnatenko committed
316

317

Igor Gnatenko's avatar
Igor Gnatenko committed
318
class Parser(argparse.ArgumentParser):
Igor Gnatenko's avatar
Igor Gnatenko committed
319 320 321 322 323
    def error(self, message):
        sys.stderr.write("error: {}\n".format(message))
        self.print_help()
        sys.exit(2)

Igor Gnatenko's avatar
Igor Gnatenko committed
324 325

def set_args(parser):
Igor Gnatenko's avatar
Igor Gnatenko committed
326 327 328 329 330 331 332 333 334
    parser.add_argument("--buildid", dest="buildid", action="store",
                        help="user build-id")
    parser.add_argument("--check-configs", dest="chk_config", action="store_true",
                        help="enable check for new CONFIG options")
    parser.add_argument("--separate-debug", dest="separate_debug", action="store_true",
                        help="separate debug kernel and main kernel")
    parser.add_argument("--without-patches", dest="patches", action="store_false",
                        help="build kernel w/o/ patches")

Igor Gnatenko's avatar
Igor Gnatenko committed
335 336

def main():
Igor Gnatenko's avatar
Igor Gnatenko committed
337 338 339 340 341 342 343 344 345 346 347 348 349
    parser = Parser(description="Make RPM from upstream linux kernel easy.")
    set_args(parser)
    args = parser.parse_args()
    options = Options(WORK_DIR)
    options.print_info()
    options.clean_tree(True)
    options.download_files()
    options.archive()
    options.make_patch()
    options.parse_spec(args)
    options.make_srpm()
    options.clean_tree(False)
    sys.exit(0)
Igor Gnatenko's avatar
Igor Gnatenko committed
350 351

if __name__ == "__main__":
Igor Gnatenko's avatar
Igor Gnatenko committed
352
    main()
Igor Gnatenko's avatar
Igor Gnatenko committed
353

Igor Gnatenko's avatar
Igor Gnatenko committed
354
# vi:et:ts=4:sw=4:sts=4