Pileus Git - ~andy/fetchmail/blob - dist-tools/shipper/shipper

   1 #!/usr/bin/env python
   2 #
   3 # shipper -- a tool for shipping software
   4
   5 import sys, os, readline, re, commands, time, glob, optparse, stat
   6
   7 #
   8 # State variables
   9 #
  10 destinations = []       # List of remote directories to update
  11 channels = ['ibiblio', 'redhat', 'freshmeat']
  12 whoami = None           # Who am I? (Used for FTP logins)
  13 date = None             # User has not yet set a date
  14 package = None          # Nor a package name
  15 homepage = None         # Nor a home page
  16 arch = None             # The machine architecture
  17 keywords = None         # Keywords for LSMs
  18 freshmeat_name = None   # Name of the project ob Freshmeat
  19 changelog = None        # Project changelog
  20 lastchange = None       # Last entry in changelog
  21 summary = None          # One-line summary of the package
  22 description = None      # Nor a description
  23
  24 indextemplate = """
  25 <?xml version="1.0" encoding="ISO-8859-1"?>
  26 <!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Transitional//EN'
  27     'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'>
  28 <html>
  29 <head>
  30 <link rel='stylesheet' href='/~esr/sitestyle.css' type='text/css' />
  31 <meta name='description' content='Resource page for %(package)s' />
  32 <meta name='generator' content='shipper' />
  33 <meta name='MSSmartTagsPreventParsing' content='TRUE' />
  34 <title>Resource page for %(package)s %(version)s</title>
  35 </head>
  36 <body>
  37
  38 <h1>Resource page for %(package)s %(version)s</td></h1>
  39
  40 <p>%(description)s</p>
  41
  42 <br />
  43 %(resourcetable)s
  44 <br />
  45
  46 <p>Last modified %(date)s.</p>
  47
  48 </div>
  49 </body>
  50 </html>
  51 """
  52 mailtemplate = """Subject: Announcing release %(version)s of %(package)s
  53
  54 Release %(version)s of %(package)s is now available at:
  55
  56         %(homepage)s
  57
  58 Here are the most recent changes:
  59
  60 %(lastchange)s
  61 --
  62                              shipper, acting for %(whoami)s
  63 """
  64
  65 # It's unpleasant that we have to include these here, but
  66 # the freshmeat release focus has to be validated even if the
  67 # user is offline and the XML-RPC service not accessible.
  68 freshmeat_focus_types = (
  69 "N/A",
  70 "Initial freshmeat announcement",
  71 "Documentation",
  72 "Code cleanup",
  73 "Minor feature enhancements",
  74 "Major feature enhancements",
  75 "Minor bugfixes",
  76 "Major bugfixes",
  77 "Minor security fixes",
  78 "Major security fixes",
  79 )
  80
  81 def croak(msg):
  82     sys.stderr.write("shipper: " + msg + "\n")
  83     sys.exit(1)
  84
  85 #
  86 # Shipping methods
  87 #
  88
  89 def do_or_die(cmd):
  90     "Wither execute a command or fail noisily"
  91     if options.verbose:
  92         print "***", cmd
  93     if os.system(cmd):
  94         croak("command '%s' failed!" % cmd)
  95
  96 def upload_or_die(cmd):
  97     if options.noupload:
  98         print cmd
  99     else:
 100         do_or_die(cmd)
 101
 102 def upload(destination, files):
 103     # Upload a file via ftp or sftp, handles
 104     print "# Uploading to %s" % destination
 105     files = filter(os.path.exists, files)
 106     if destination.startswith("ftp://"):
 107         destination = destination[6:].split("/")
 108         host = destination.pop(0)
 109         directory = "/".join(destination)
 110         commands = ["lftp", "open -u anonymous," + whoami + " " + host + "\n"]
 111         if directory:
 112             commands.append("cd " + directory + "\n")
 113         commands.append("mput " + " ".join(files) + "\n")
 114         commands.append("close\n")
 115         if options.noupload:
 116             print "".join(commands)
 117         else:
 118             pfp = os.popen(commands.pop(0), "w")
 119             pfp.writelines(commands)
 120             pfp.close()
 121     elif destination.find("::") > -1:
 122         upload_or_die("rsync " + " ".join(files) + " " + destination)
 123     elif destination.find(":") > -1:
 124         (host, directory) = destination.split(":")
 125         for file in files:
 126             # This is a really ugly way to deal with the problem
 127             # of write-protected files in the remote directory.
 128             # Unfortunately, sftp(1) is rather brain-dead -- no
 129             # way to ignore failure on a remove, and refuses to
 130             # do renames with an obscure error message.
 131             remote = os.path.join(directory, package, file)
 132             upload_or_die("scp " + file + " " + host + ":" + remote+".new;")
 133             upload_or_die("ssh %s 'mv -f %s.new %s'" % (host, remote, remote))
 134     else:
 135         sys.stderr.write("Don't know what to do with destination %s!")
 136
 137 def freshmeat_ship(manifest):
 138     "Ship a specified update to freshmeat."
 139     if options.verbose:
 140         print "Announcing to freshmeat..."
 141     upload_or_die("freshmeat-submit <" + manifest[0])
 142
 143 #
 144 # Metadata extraction
 145 #
 146
 147 def grep(pattern, file):
 148     "Mine for a specified pattern in a file."
 149     fp = open(file)
 150     try:
 151         while True:
 152             line = fp.readline()
 153             if not line:
 154                 return None
 155             m = re.search(pattern, line)
 156             if m:
 157                 return m.group(1)
 158     finally:
 159         fp.close()
 160     return None
 161
 162 class Specfile:
 163     def __init__(self, filename):
 164         self.filename = filename
 165         self.type = None
 166         if filename.endswith(".spec"):
 167             self.type = "RPM"
 168             self.package = self.extract("Name")
 169             self.version = self.extract("Version")
 170             self.homepage = self.extract("URL")
 171             self.summary = self.extract("Summary")
 172             self.arch = self.extract("BuildArch") or commands.getoutput("rpm --showrc | sed -n '/^build arch/s/.* //p'")
 173             self.description = self.rpm_get_multiline("description")
 174             self.changelog = self.rpm_get_multiline("changelog")
 175         elif filename == "control":
 176             self.type = "deb"
 177             self.name = self.extract("Package")
 178             self.version = self.extract("Version").split("-")[0]
 179             self.homepage = self.extract("XBS-Home-Page")
 180             self.summary = self.extract("Description")
 181             self.arch = self.extract("Architecture")
 182             if not self.arch:
 183                 croak("this control file lacks an Architecture field")
 184             # FIXME: parse Debian description entries and changelog file
 185             self.description = self.changelog = None
 186     def extract(self, fld):
 187         "Extract a one-line field, possibly embedded as a magic comment."
 188         if self.type == "RPM":
 189             return grep("^#?"+fld+":\s*(.*)", self.filename)
 190         elif self.type == "deb":
 191             return grep("^(?:XBS-)?"+fld+": (.*)", self.filename)
 192     def rpm_get_multiline(self, fieldname):
 193         "Grab everything from leader line to just before the next leader line."
 194         global desc
 195         fp = open(self.filename)
 196         desc = ""
 197         gather = False
 198         while True:
 199             line = fp.readline()
 200             if not line:
 201                 break
 202             # Pick up fieldnames *without* translation options.
 203             if line.strip() == "%" + fieldname:
 204                 gather = True
 205                 continue
 206             elif line[0] == "%":
 207                 gather = False
 208             if gather:
 209                 desc += line
 210         fp.close()
 211         if desc:
 212             return desc.strip() + "\n"
 213         else:
 214             return None
 215 #
 216 # Main sequence
 217 #
 218
 219 try:
 220     #
 221     # Process options
 222     #
 223
 224     parser = optparse.OptionParser(usage="%prog: [-h] [-n] [-f] [-v]")
 225     parser.add_option("-v", "--verbose",
 226                       action="store_true", dest="verbose", default=False,
 227                       help="print progress messages to stdout")
 228     parser.add_option("-n", "--noupload",
 229                       action="store_true", dest="noupload", default=False,
 230                       help="don't do uploads, just build deliverables")
 231     parser.add_option("-N", "--nobuild",
 232                       action="store_true", dest="nobuild", default=False,
 233                       help="dump configuration only, no builds or uploads")
 234     parser.add_option("-f", "--force",
 235                       action="store_true", dest="force", default=False,
 236                       help="force rebuilding of all local deliverables")
 237     (options, args) = parser.parse_args()
 238
 239     #
 240     # Extract metadata and compute control information
 241     #
 242
 243     def disable(s): channels.remove(s)
 244
 245     # Security check, don't let an attacker elevate privileges
 246     def securecheck(file):
 247         if stat.S_IMODE(os.stat(file).st_mode) & 00002:
 248             croak("%s must not be world-writeable!" % file)
 249
 250     # Read in variable overrides
 251     securecheck(".")
 252     home_profile = os.path.join(os.getenv('HOME'), ".shipper")
 253     if os.path.exists(home_profile):
 254         securecheck(home_profile)
 255         execfile(home_profile)
 256     here_profile = ".shipper"
 257     if os.path.exists(here_profile):
 258         securecheck(here_profile)
 259         execfile(here_profile)
 260
 261     # Set various sensible defaults
 262     if not whoami:
 263         whoami = os.getenv('USERNAME') + "@" + os.getenv('HOSTNAME')
 264
 265     # Where to get the metadata
 266     specfiles = glob.glob("*.spec")
 267     if len(specfiles) == 1:
 268         metadata = Specfile(specfiles[0])
 269     elif os.path.exists("control"):
 270         metadata = Specfile("control")
 271     else:
 272         croak("must be exactly one RPM or dpkg specfile in the directory!")
 273
 274     # Get the package name
 275     if not package:
 276         package = metadata.package
 277     if not package:
 278         croak("can't get package name!")
 279
 280     # Extract the package vers from the specfile or Makefile
 281     specvers = metadata.version
 282     makevers = None
 283     if os.path.exists("Makefile"):
 284         makevers = grep("^VERS[A-Z]* *= *(.*)", "Makefile")
 285         # Maybe it's a shell command intended to extract version from specfile
 286         if makevers and makevers[0] == '$':
 287             makevers = commands.getoutput(makevers[7:-1])
 288     if specvers != makevers:
 289         croak("specfile version %s != Makefile version %s"%(specvers,makevers))
 290     elif specvers == None:
 291         croak("can't get package version")
 292     elif specvers[0] not in "0123456789":
 293         croak("package version %s appears garbled" % specvers)
 294     else:
 295         version = specvers
 296
 297     # Specfiles may set their own destinations
 298     local_destinations = metadata.extract("Destinations")
 299     if local_destinations:
 300         local_destinations = map(lambda x: x.strip(), local_destinations.split(","))
 301         destinations += local_destinations
 302     if not destinations:
 303         print "warning: destinations empty, shipping to public channels only."
 304
 305     print"# Uploading version %s of %s" % (version, package)
 306
 307     # Extract remaining variables for templating
 308     if not homepage:
 309         homepage = metadata.homepage
 310     if not date:
 311         date = time.asctime()
 312     if not summary:
 313         summary = metadata.summary
 314     if not description:
 315         description = metadata.description
 316     if not arch:
 317         arch = metadata.arch
 318     if not keywords:
 319         keywords = metadata.extract("Keywords")
 320     if not freshmeat_name:
 321         freshmeat_name = metadata.extract("Freshmeat-Name")
 322
 323     # Finally, derive the change log and lastchange entry;
 324     # we'll need the latter for freshmeat.net
 325     freshmeat_lastchange = lastchange = changelog = None
 326     # ChangeLog, if present, takes precedence;
 327     # we assume if both are present that the specfile log is about packaging.
 328     if os.path.exists("ChangeLog"):
 329         ifp = open("ChangeLog", "r")
 330         changelog = ifp.read()
 331         ifp.close()
 332         lastchange = ""
 333         for line in changelog.split("\n"):
 334             while line.strip() or not "*" in lastchange:
 335                 lastchange += line + "\n"
 336             else:
 337                 break
 338         # freshmeat.net doesn't like bulleted items in a changes field.
 339         freshmeat_lastchange = "See the ChangeLog file for recent changes."
 340     elif metadata.changelog:
 341         changelog = metadata.changelog
 342         lastchange = ""
 343         for line in changelog.split("\n"):
 344             if not lastchange and (not line.strip() or line[0] == '*'):
 345                 continue
 346             elif line.strip():
 347                 lastchange += line + "\n"
 348             else:
 349                 break
 350         # This usually produces a lastchange entry that freshmeat will take.
 351         freshmeat_lastchange = lastchange
 352
 353     #
 354     # Now compute the names of deliverables
 355     #
 356
 357     # These are all potential deliverable files that include the version number
 358     tarball   = package + "-" + version + ".tar.gz"
 359     srcrpm    = package + "-" + version + "-1.src.rpm"
 360     binrpm    = package + "-" + version + "-1." + arch + ".rpm"
 361     zip       = package + "-" + version + ".zip"
 362     lsm       = package + "-" + version + ".lsm"
 363
 364     # Map web deliverables to explanations for the resource table
 365     # Stuff not included here: ANNOUNCE.EMAIL, ANNOUNCE.FRESHMEAT, lsm.
 366     stock_deliverables = [
 367         ("README",      "roadmap file"),
 368         (tarball,       "source tarball"),
 369         (zip,           "ZIP archive"),
 370         (binrpm,        "installable RPM"),     # Generated
 371         (srcrpm,        "source RPM"),          # Generated
 372         ("ChangeLog",   "change log"),
 373         ("CHANGES",     "change log"),          # Generated
 374         ("NEWS",        "Project news"),
 375         ("HISTORY",     "Project history"),
 376         ("BUGS",        "Known bugs"),
 377         ("TODO",        "To-do file"),
 378         ]
 379
 380     #
 381     # Might be time to dump
 382     #
 383     if options.nobuild:
 384         for variable in ('destinations', 'channels', 'whoami', 'date',
 385                          'package', 'homepage', 'arch', 'keywords', \
 386                          'freshmeat_name', 'summary'):
 387             print "%s = %s" % (variable, `eval(variable)`)
 388         for variable in ('description', 'changelog', 'lastchange', 'mailtemplate', 'indextemplate'):
 389             if not eval(variable):
 390                 print "No %s" % variable
 391             else:
 392                 print "%s = <<EOF\n%sEOF" % (variable, eval(variable))
 393         sys.exit(0)
 394     #
 395     # Build deliverables
 396     #
 397
 398     suppress = " >/dev/null 2>&1"
 399     if options.verbose:
 400         suppress = ""
 401
 402     # Sanity checks
 403     if not os.path.exists(tarball):
 404         croak("no tarball %s!" % tarball)
 405     if metadata.type == "RPM" and not metadata.extract("BuildRoot"):
 406         croak("specfile %s doesn't have a BuildRoot!" % metadata.filename)
 407
 408     def newer(f1, f2):
 409         return os.path.exists(f1) and (os.stat(f1).st_mtime > os.stat(f2).st_mtime)
 410
 411     # Compute the deliverables, we need this even if not rebuilding the index
 412     web_deliverables = []
 413     # Anything in the list of standard deliverables is eligible.
 414     for (file, explanation) in stock_deliverables:
 415         if os.path.exists(file):
 416             web_deliverables.append((file, explanation))
 417     # So is anything with an HTML extendion
 418     for file in glob.glob('*.html'):
 419         if file == 'index.html':
 420             continue
 421         stem = file[:-4]
 422         for ext in ("man", "1", "2", "3", "4", "5", "6", "7", "8", "9", "xml"):
 423             if os.path.exists(stem + ext):
 424                 explanation = "HTML rendering of " + stem + ext
 425                 break
 426         else:
 427             explanation = "HTML page."
 428         web_deliverables.append((file, explanation))
 429     # Compute final deliverables
 430     deliverables = map(lambda x: x[0], web_deliverables)+["index.html"]
 431
 432     try:
 433         delete_at_end = []
 434
 435         # RPMs first.
 436         if options.force or \
 437                (not os.path.exists(binrpm) or not os.path.exists(srcrpm)):
 438             print "# Building RPMs..."
 439             if newer(srcrpm, tarball) and newer(binrpm, tarball):
 440                 print "RPMs are up to date"
 441             else:
 442                 do_or_die("buildrpms %s %s" % (tarball, suppress))
 443                 delete_at_end.append(srcrpm)
 444                 delete_at_end.append(binrpm)
 445
 446         # Next, the LSM if needed
 447         if 'ibiblio' in channels and \
 448                (options.force or not os.path.exists(lsm)):
 449             print "# Building LSM..."
 450             if keywords:
 451                 do_or_die("rpm2lsm -k '"+keywords+"' "+binrpm+" >"+lsm)
 452             else:
 453                 print "# Warning: LSM being built with no keywords."
 454                 do_or_die("rpm2lsm " + binrpm + ">" + lsm)
 455             delete_at_end.append(lsm)
 456
 457         # Next the index page if it doesn't exist.
 458         if homepage and (options.force or not os.path.exists("index.html")):
 459             print "# Building index page..."
 460             # Now build the resource table
 461             resourcetable = '<table border="1" align="center" summary="Downloadable resources">\n'
 462             for (file, explanation) in web_deliverables:
 463                 resourcetable += "<tr><td><a href='%s'>%s</a></td><td>%s</td></tr>\n" % (file,file,explanation)
 464             resourcetable += "</table>"
 465             # OK, now build the index page itself
 466             ofp = open("index.html", "w")
 467             ofp.write(indextemplate % globals())
 468             ofp.close()
 469             delete_at_end.append("index.html")
 470
 471         # Next the CHANGES file.  Build this only if (a) there is no ChangeLog,
 472         # and (b) there is a specfile %changelog.
 473         if not os.path.exists("ChangeLog") and \
 474                (options.force or not os.path.exists("CHANGES")) and changelog:
 475             print "# Building CHANGES..."
 476             ofp = open("CHANGES", "w")
 477             ofp.write("                     Changelog for " + package + "\n\n")
 478             ofp.write(changelog)
 479             ofp.close()
 480             delete_at_end.append("CHANGES")
 481
 482         # The freshmeat announcement
 483         if 'freshmeat' in channels \
 484                and options.force or not os.path.exists("ANNOUNCE.FRESHMEAT"):
 485             print "# Building ANNOUNCE.FRESHMEAT..."
 486             if not homepage:
 487                 print "# Can't announce to freshmeat without a primary website!"
 488             elif not lastchange:
 489                 print "# Can't announce to freshmeat without a changes field!"
 490             else:
 491                 while True:
 492                     focus = raw_input("# freshmeat.net release focus (? for list): ")
 493                     if focus == '?':
 494                         i = 0
 495                         for f in freshmeat_focus_types:
 496                             print "%d: %s" % (i, f)
 497                             i += 1
 498                     elif focus in "0123456789":
 499                         print "# OK:", freshmeat_focus_types[int(focus)]
 500                         break
 501                     elif focus.lower() in map(lambda x: x.lower(), freshmeat_focus_types):
 502                         break
 503                     else:
 504                         croak("not a valid freshmeat.net release focus!")
 505                 ofp = open("ANNOUNCE.FRESHMEAT", "w")
 506                 ofp.write("Project: %s\n"%(freshmeat_name or package))
 507                 ofp.write("Version: %s\n"% version)
 508                 ofp.write("Release-Focus: %s\n" % focus)
 509                 ofp.write("Home-Page-URL: %s\n" % homepage)
 510                 if os.path.exists(tarball):
 511                     ofp.write("Gzipped-Tar-URL: %s\n" % os.path.join(homepage,tarball))
 512                 if os.path.exists(zip):
 513                     ofp.write("Zipped-Tar-URL: %s\n" % os.path.join(homepage, zip))
 514                 if os.path.exists("CHANGES"):
 515                     ofp.write("Changelog-URL: %s\n" % os.path.join(homepage, "CHANGES"))
 516                 if os.path.exists(binrpm):
 517                     ofp.write("RPM-URL: %s\n" % os.path.join(homepage, binrpm))
 518                 # freshmeat.net doesn't like bulleted entries.
 519                 freshmeatlog = lastchange[2:].replace("\n  ", "\n")
 520                 ofp.write("\n" + freshmeatlog)
 521                 ofp.close()
 522                 delete_at_end.append("ANNOUNCE.FRESHMEAT")
 523
 524         # Finally, email notification
 525         if filter(lambda x: x.startswith("mailto:"), destinations) \
 526                and (options.force or not os.path.exists("ANNOUNCE.EMAIL")):
 527             print "# Building ANNOUNCE.EMAIL..."
 528             ofp = open("ANNOUNCE.EMAIL", "w")
 529             ofp.write(mailtemplate % globals())
 530             ofp.close()
 531             delete_at_end.append("ANNOUNCE.FRESHMEAT")
 532
 533         #
 534         # Now actually ship
 535         #
 536
 537         # Shipping methods, locations, and deliverables for public channels.
 538         hardwired = {
 539             'freshmeat' : (lambda: freshmeat_ship(("ANNOUNCE.FRESHMEAT",))),
 540             'ibiblio'   : (lambda: upload("ftp://ibiblio.org/incoming/linux",
 541                                           (tarball, binrpm, srcrpm, lsm))),
 542             'redhat'    : (lambda: upload("ftp://incoming.redhat.com/libc6",
 543                                           (tarball, binrpm, srcrpm))),
 544         }
 545
 546
 547         # First ship to private channels.  Order is important here, we
 548         # need to hit the user's primary website first so everything
 549         # will be in place when announcements are generated.
 550         for destination in destinations:
 551             if destination.startswith("ftp:"):
 552                 upload(destination, (tarball, binrpm, srcrpm,))
 553             elif destination.startswith("mailto:"):
 554                 print "# Mailing to %s" % destination
 555                 command = "sendmail -i -oem -f %s %s <ANNOUNCE.EMAIL" % (whoami, destination[7:])
 556                 if options.noupload:
 557                     print command
 558                 else:
 559                     do_or_die(command)
 560             else:
 561                 upload(destination, deliverables)
 562
 563         # Now ship to public channels
 564         for channel in channels:
 565             print "# Shipping to public channel", channel
 566             apply(hardwired[channel])
 567     finally:
 568         cleanup = "rm -f " + " ".join(delete_at_end)
 569         if options.noupload:
 570             print cleanup
 571         else:
 572             for file in delete_at_end:
 573                 os.system(cleanup)
 574     print "# Done"
 575 except KeyboardInterrupt:
 576     print "# Bye!"
 577
 578
 579
 580 # The following sets edit modes for GNU EMACS
 581 # Local Variables:
 582 # mode:python
 583 # End: