Pileus Git - ~andy/git/blob - git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re, shutil
  14
  15 verbose = False
  16
  17 # Only labels/tags matching this will be imported/exported
  18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = ["p4"]
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += ["-u",user]
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += ["-P", password]
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += ["-p", port]
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += ["-H", host]
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += ["-c", client]
  48
  49
  50     if isinstance(cmd,basestring):
  51         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  52     else:
  53         real_cmd += cmd
  54     return real_cmd
  55
  56 def chdir(dir):
  57     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  58     # not using the shell, we have to set it ourselves.  This path could
  59     # be relative, so go there first, then figure out where we ended up.
  60     os.chdir(dir)
  61     os.environ['PWD'] = os.getcwd()
  62
  63 def die(msg):
  64     if verbose:
  65         raise Exception(msg)
  66     else:
  67         sys.stderr.write(msg + "\n")
  68         sys.exit(1)
  69
  70 def write_pipe(c, stdin):
  71     if verbose:
  72         sys.stderr.write('Writing pipe: %s\n' % str(c))
  73
  74     expand = isinstance(c,basestring)
  75     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  76     pipe = p.stdin
  77     val = pipe.write(stdin)
  78     pipe.close()
  79     if p.wait():
  80         die('Command failed: %s' % str(c))
  81
  82     return val
  83
  84 def p4_write_pipe(c, stdin):
  85     real_cmd = p4_build_cmd(c)
  86     return write_pipe(real_cmd, stdin)
  87
  88 def read_pipe(c, ignore_error=False):
  89     if verbose:
  90         sys.stderr.write('Reading pipe: %s\n' % str(c))
  91
  92     expand = isinstance(c,basestring)
  93     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  94     pipe = p.stdout
  95     val = pipe.read()
  96     if p.wait() and not ignore_error:
  97         die('Command failed: %s' % str(c))
  98
  99     return val
 100
 101 def p4_read_pipe(c, ignore_error=False):
 102     real_cmd = p4_build_cmd(c)
 103     return read_pipe(real_cmd, ignore_error)
 104
 105 def read_pipe_lines(c):
 106     if verbose:
 107         sys.stderr.write('Reading pipe: %s\n' % str(c))
 108
 109     expand = isinstance(c, basestring)
 110     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 111     pipe = p.stdout
 112     val = pipe.readlines()
 113     if pipe.close() or p.wait():
 114         die('Command failed: %s' % str(c))
 115
 116     return val
 117
 118 def p4_read_pipe_lines(c):
 119     """Specifically invoke p4 on the command supplied. """
 120     real_cmd = p4_build_cmd(c)
 121     return read_pipe_lines(real_cmd)
 122
 123 def system(cmd):
 124     expand = isinstance(cmd,basestring)
 125     if verbose:
 126         sys.stderr.write("executing %s\n" % str(cmd))
 127     subprocess.check_call(cmd, shell=expand)
 128
 129 def p4_system(cmd):
 130     """Specifically invoke p4 as the system command. """
 131     real_cmd = p4_build_cmd(cmd)
 132     expand = isinstance(real_cmd, basestring)
 133     subprocess.check_call(real_cmd, shell=expand)
 134
 135 def p4_integrate(src, dest):
 136     p4_system(["integrate", "-Dt", src, dest])
 137
 138 def p4_sync(f, *options):
 139     p4_system(["sync"] + list(options) + [f])
 140
 141 def p4_add(f):
 142     p4_system(["add", f])
 143
 144 def p4_delete(f):
 145     p4_system(["delete", f])
 146
 147 def p4_edit(f):
 148     p4_system(["edit", f])
 149
 150 def p4_revert(f):
 151     p4_system(["revert", f])
 152
 153 def p4_reopen(type, file):
 154     p4_system(["reopen", "-t", type, file])
 155
 156 #
 157 # Canonicalize the p4 type and return a tuple of the
 158 # base type, plus any modifiers.  See "p4 help filetypes"
 159 # for a list and explanation.
 160 #
 161 def split_p4_type(p4type):
 162
 163     p4_filetypes_historical = {
 164         "ctempobj": "binary+Sw",
 165         "ctext": "text+C",
 166         "cxtext": "text+Cx",
 167         "ktext": "text+k",
 168         "kxtext": "text+kx",
 169         "ltext": "text+F",
 170         "tempobj": "binary+FSw",
 171         "ubinary": "binary+F",
 172         "uresource": "resource+F",
 173         "uxbinary": "binary+Fx",
 174         "xbinary": "binary+x",
 175         "xltext": "text+Fx",
 176         "xtempobj": "binary+Swx",
 177         "xtext": "text+x",
 178         "xunicode": "unicode+x",
 179         "xutf16": "utf16+x",
 180     }
 181     if p4type in p4_filetypes_historical:
 182         p4type = p4_filetypes_historical[p4type]
 183     mods = ""
 184     s = p4type.split("+")
 185     base = s[0]
 186     mods = ""
 187     if len(s) > 1:
 188         mods = s[1]
 189     return (base, mods)
 190
 191 #
 192 # return the raw p4 type of a file (text, text+ko, etc)
 193 #
 194 def p4_type(file):
 195     results = p4CmdList(["fstat", "-T", "headType", file])
 196     return results[0]['headType']
 197
 198 #
 199 # Given a type base and modifier, return a regexp matching
 200 # the keywords that can be expanded in the file
 201 #
 202 def p4_keywords_regexp_for_type(base, type_mods):
 203     if base in ("text", "unicode", "binary"):
 204         kwords = None
 205         if "ko" in type_mods:
 206             kwords = 'Id|Header'
 207         elif "k" in type_mods:
 208             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 209         else:
 210             return None
 211         pattern = r"""
 212             \$              # Starts with a dollar, followed by...
 213             (%s)            # one of the keywords, followed by...
 214             (:[^$]+)?       # possibly an old expansion, followed by...
 215             \$              # another dollar
 216             """ % kwords
 217         return pattern
 218     else:
 219         return None
 220
 221 #
 222 # Given a file, return a regexp matching the possible
 223 # RCS keywords that will be expanded, or None for files
 224 # with kw expansion turned off.
 225 #
 226 def p4_keywords_regexp_for_file(file):
 227     if not os.path.exists(file):
 228         return None
 229     else:
 230         (type_base, type_mods) = split_p4_type(p4_type(file))
 231         return p4_keywords_regexp_for_type(type_base, type_mods)
 232
 233 def setP4ExecBit(file, mode):
 234     # Reopens an already open file and changes the execute bit to match
 235     # the execute bit setting in the passed in mode.
 236
 237     p4Type = "+x"
 238
 239     if not isModeExec(mode):
 240         p4Type = getP4OpenedType(file)
 241         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 242         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 243         if p4Type[-1] == "+":
 244             p4Type = p4Type[0:-1]
 245
 246     p4_reopen(p4Type, file)
 247
 248 def getP4OpenedType(file):
 249     # Returns the perforce file type for the given file.
 250
 251     result = p4_read_pipe(["opened", file])
 252     match = re.match(".*\((.+)\)\r?$", result)
 253     if match:
 254         return match.group(1)
 255     else:
 256         die("Could not determine file type for %s (result: '%s')" % (file, result))
 257
 258 # Return the set of all p4 labels
 259 def getP4Labels(depotPaths):
 260     labels = set()
 261     if isinstance(depotPaths,basestring):
 262         depotPaths = [depotPaths]
 263
 264     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 265         label = l['label']
 266         labels.add(label)
 267
 268     return labels
 269
 270 # Return the set of all git tags
 271 def getGitTags():
 272     gitTags = set()
 273     for line in read_pipe_lines(["git", "tag"]):
 274         tag = line.strip()
 275         gitTags.add(tag)
 276     return gitTags
 277
 278 def diffTreePattern():
 279     # This is a simple generator for the diff tree regex pattern. This could be
 280     # a class variable if this and parseDiffTreeEntry were a part of a class.
 281     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 282     while True:
 283         yield pattern
 284
 285 def parseDiffTreeEntry(entry):
 286     """Parses a single diff tree entry into its component elements.
 287
 288     See git-diff-tree(1) manpage for details about the format of the diff
 289     output. This method returns a dictionary with the following elements:
 290
 291     src_mode - The mode of the source file
 292     dst_mode - The mode of the destination file
 293     src_sha1 - The sha1 for the source file
 294     dst_sha1 - The sha1 fr the destination file
 295     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 296     status_score - The score for the status (applicable for 'C' and 'R'
 297                    statuses). This is None if there is no score.
 298     src - The path for the source file.
 299     dst - The path for the destination file. This is only present for
 300           copy or renames. If it is not present, this is None.
 301
 302     If the pattern is not matched, None is returned."""
 303
 304     match = diffTreePattern().next().match(entry)
 305     if match:
 306         return {
 307             'src_mode': match.group(1),
 308             'dst_mode': match.group(2),
 309             'src_sha1': match.group(3),
 310             'dst_sha1': match.group(4),
 311             'status': match.group(5),
 312             'status_score': match.group(6),
 313             'src': match.group(7),
 314             'dst': match.group(10)
 315         }
 316     return None
 317
 318 def isModeExec(mode):
 319     # Returns True if the given git mode represents an executable file,
 320     # otherwise False.
 321     return mode[-3:] == "755"
 322
 323 def isModeExecChanged(src_mode, dst_mode):
 324     return isModeExec(src_mode) != isModeExec(dst_mode)
 325
 326 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 327
 328     if isinstance(cmd,basestring):
 329         cmd = "-G " + cmd
 330         expand = True
 331     else:
 332         cmd = ["-G"] + cmd
 333         expand = False
 334
 335     cmd = p4_build_cmd(cmd)
 336     if verbose:
 337         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 338
 339     # Use a temporary file to avoid deadlocks without
 340     # subprocess.communicate(), which would put another copy
 341     # of stdout into memory.
 342     stdin_file = None
 343     if stdin is not None:
 344         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 345         if isinstance(stdin,basestring):
 346             stdin_file.write(stdin)
 347         else:
 348             for i in stdin:
 349                 stdin_file.write(i + '\n')
 350         stdin_file.flush()
 351         stdin_file.seek(0)
 352
 353     p4 = subprocess.Popen(cmd,
 354                           shell=expand,
 355                           stdin=stdin_file,
 356                           stdout=subprocess.PIPE)
 357
 358     result = []
 359     try:
 360         while True:
 361             entry = marshal.load(p4.stdout)
 362             if cb is not None:
 363                 cb(entry)
 364             else:
 365                 result.append(entry)
 366     except EOFError:
 367         pass
 368     exitCode = p4.wait()
 369     if exitCode != 0:
 370         entry = {}
 371         entry["p4ExitCode"] = exitCode
 372         result.append(entry)
 373
 374     return result
 375
 376 def p4Cmd(cmd):
 377     list = p4CmdList(cmd)
 378     result = {}
 379     for entry in list:
 380         result.update(entry)
 381     return result;
 382
 383 def p4Where(depotPath):
 384     if not depotPath.endswith("/"):
 385         depotPath += "/"
 386     depotPath = depotPath + "..."
 387     outputList = p4CmdList(["where", depotPath])
 388     output = None
 389     for entry in outputList:
 390         if "depotFile" in entry:
 391             if entry["depotFile"] == depotPath:
 392                 output = entry
 393                 break
 394         elif "data" in entry:
 395             data = entry.get("data")
 396             space = data.find(" ")
 397             if data[:space] == depotPath:
 398                 output = entry
 399                 break
 400     if output == None:
 401         return ""
 402     if output["code"] == "error":
 403         return ""
 404     clientPath = ""
 405     if "path" in output:
 406         clientPath = output.get("path")
 407     elif "data" in output:
 408         data = output.get("data")
 409         lastSpace = data.rfind(" ")
 410         clientPath = data[lastSpace + 1:]
 411
 412     if clientPath.endswith("..."):
 413         clientPath = clientPath[:-3]
 414     return clientPath
 415
 416 def currentGitBranch():
 417     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 418
 419 def isValidGitDir(path):
 420     if (os.path.exists(path + "/HEAD")
 421         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 422         return True;
 423     return False
 424
 425 def parseRevision(ref):
 426     return read_pipe("git rev-parse %s" % ref).strip()
 427
 428 def branchExists(ref):
 429     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 430                      ignore_error=True)
 431     return len(rev) > 0
 432
 433 def extractLogMessageFromGitCommit(commit):
 434     logMessage = ""
 435
 436     ## fixme: title is first line of commit, not 1st paragraph.
 437     foundTitle = False
 438     for log in read_pipe_lines("git cat-file commit %s" % commit):
 439        if not foundTitle:
 440            if len(log) == 1:
 441                foundTitle = True
 442            continue
 443
 444        logMessage += log
 445     return logMessage
 446
 447 def extractSettingsGitLog(log):
 448     values = {}
 449     for line in log.split("\n"):
 450         line = line.strip()
 451         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 452         if not m:
 453             continue
 454
 455         assignments = m.group(1).split (':')
 456         for a in assignments:
 457             vals = a.split ('=')
 458             key = vals[0].strip()
 459             val = ('='.join (vals[1:])).strip()
 460             if val.endswith ('\"') and val.startswith('"'):
 461                 val = val[1:-1]
 462
 463             values[key] = val
 464
 465     paths = values.get("depot-paths")
 466     if not paths:
 467         paths = values.get("depot-path")
 468     if paths:
 469         values['depot-paths'] = paths.split(',')
 470     return values
 471
 472 def gitBranchExists(branch):
 473     proc = subprocess.Popen(["git", "rev-parse", branch],
 474                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 475     return proc.wait() == 0;
 476
 477 _gitConfig = {}
 478 def gitConfig(key, args = None): # set args to "--bool", for instance
 479     if not _gitConfig.has_key(key):
 480         argsFilter = ""
 481         if args != None:
 482             argsFilter = "%s " % args
 483         cmd = "git config %s%s" % (argsFilter, key)
 484         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 485     return _gitConfig[key]
 486
 487 def gitConfigList(key):
 488     if not _gitConfig.has_key(key):
 489         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 490     return _gitConfig[key]
 491
 492 def p4BranchesInGit(branchesAreInRemotes = True):
 493     branches = {}
 494
 495     cmdline = "git rev-parse --symbolic "
 496     if branchesAreInRemotes:
 497         cmdline += " --remotes"
 498     else:
 499         cmdline += " --branches"
 500
 501     for line in read_pipe_lines(cmdline):
 502         line = line.strip()
 503
 504         ## only import to p4/
 505         if not line.startswith('p4/') or line == "p4/HEAD":
 506             continue
 507         branch = line
 508
 509         # strip off p4
 510         branch = re.sub ("^p4/", "", line)
 511
 512         branches[branch] = parseRevision(line)
 513     return branches
 514
 515 def findUpstreamBranchPoint(head = "HEAD"):
 516     branches = p4BranchesInGit()
 517     # map from depot-path to branch name
 518     branchByDepotPath = {}
 519     for branch in branches.keys():
 520         tip = branches[branch]
 521         log = extractLogMessageFromGitCommit(tip)
 522         settings = extractSettingsGitLog(log)
 523         if settings.has_key("depot-paths"):
 524             paths = ",".join(settings["depot-paths"])
 525             branchByDepotPath[paths] = "remotes/p4/" + branch
 526
 527     settings = None
 528     parent = 0
 529     while parent < 65535:
 530         commit = head + "~%s" % parent
 531         log = extractLogMessageFromGitCommit(commit)
 532         settings = extractSettingsGitLog(log)
 533         if settings.has_key("depot-paths"):
 534             paths = ",".join(settings["depot-paths"])
 535             if branchByDepotPath.has_key(paths):
 536                 return [branchByDepotPath[paths], settings]
 537
 538         parent = parent + 1
 539
 540     return ["", settings]
 541
 542 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 543     if not silent:
 544         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 545                % localRefPrefix)
 546
 547     originPrefix = "origin/p4/"
 548
 549     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 550         line = line.strip()
 551         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 552             continue
 553
 554         headName = line[len(originPrefix):]
 555         remoteHead = localRefPrefix + headName
 556         originHead = line
 557
 558         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 559         if (not original.has_key('depot-paths')
 560             or not original.has_key('change')):
 561             continue
 562
 563         update = False
 564         if not gitBranchExists(remoteHead):
 565             if verbose:
 566                 print "creating %s" % remoteHead
 567             update = True
 568         else:
 569             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 570             if settings.has_key('change') > 0:
 571                 if settings['depot-paths'] == original['depot-paths']:
 572                     originP4Change = int(original['change'])
 573                     p4Change = int(settings['change'])
 574                     if originP4Change > p4Change:
 575                         print ("%s (%s) is newer than %s (%s). "
 576                                "Updating p4 branch from origin."
 577                                % (originHead, originP4Change,
 578                                   remoteHead, p4Change))
 579                         update = True
 580                 else:
 581                     print ("Ignoring: %s was imported from %s while "
 582                            "%s was imported from %s"
 583                            % (originHead, ','.join(original['depot-paths']),
 584                               remoteHead, ','.join(settings['depot-paths'])))
 585
 586         if update:
 587             system("git update-ref %s %s" % (remoteHead, originHead))
 588
 589 def originP4BranchesExist():
 590         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 591
 592 def p4ChangesForPaths(depotPaths, changeRange):
 593     assert depotPaths
 594     cmd = ['changes']
 595     for p in depotPaths:
 596         cmd += ["%s...%s" % (p, changeRange)]
 597     output = p4_read_pipe_lines(cmd)
 598
 599     changes = {}
 600     for line in output:
 601         changeNum = int(line.split(" ")[1])
 602         changes[changeNum] = True
 603
 604     changelist = changes.keys()
 605     changelist.sort()
 606     return changelist
 607
 608 def p4PathStartsWith(path, prefix):
 609     # This method tries to remedy a potential mixed-case issue:
 610     #
 611     # If UserA adds  //depot/DirA/file1
 612     # and UserB adds //depot/dira/file2
 613     #
 614     # we may or may not have a problem. If you have core.ignorecase=true,
 615     # we treat DirA and dira as the same directory
 616     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 617     if ignorecase:
 618         return path.lower().startswith(prefix.lower())
 619     return path.startswith(prefix)
 620
 621 def getClientSpec():
 622     """Look at the p4 client spec, create a View() object that contains
 623        all the mappings, and return it."""
 624
 625     specList = p4CmdList("client -o")
 626     if len(specList) != 1:
 627         die('Output from "client -o" is %d lines, expecting 1' %
 628             len(specList))
 629
 630     # dictionary of all client parameters
 631     entry = specList[0]
 632
 633     # just the keys that start with "View"
 634     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 635
 636     # hold this new View
 637     view = View()
 638
 639     # append the lines, in order, to the view
 640     for view_num in range(len(view_keys)):
 641         k = "View%d" % view_num
 642         if k not in view_keys:
 643             die("Expected view key %s missing" % k)
 644         view.append(entry[k])
 645
 646     return view
 647
 648 def getClientRoot():
 649     """Grab the client directory."""
 650
 651     output = p4CmdList("client -o")
 652     if len(output) != 1:
 653         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 654
 655     entry = output[0]
 656     if "Root" not in entry:
 657         die('Client has no "Root"')
 658
 659     return entry["Root"]
 660
 661 class Command:
 662     def __init__(self):
 663         self.usage = "usage: %prog [options]"
 664         self.needsGit = True
 665         self.verbose = False
 666
 667 class P4UserMap:
 668     def __init__(self):
 669         self.userMapFromPerforceServer = False
 670         self.myP4UserId = None
 671
 672     def p4UserId(self):
 673         if self.myP4UserId:
 674             return self.myP4UserId
 675
 676         results = p4CmdList("user -o")
 677         for r in results:
 678             if r.has_key('User'):
 679                 self.myP4UserId = r['User']
 680                 return r['User']
 681         die("Could not find your p4 user id")
 682
 683     def p4UserIsMe(self, p4User):
 684         # return True if the given p4 user is actually me
 685         me = self.p4UserId()
 686         if not p4User or p4User != me:
 687             return False
 688         else:
 689             return True
 690
 691     def getUserCacheFilename(self):
 692         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 693         return home + "/.gitp4-usercache.txt"
 694
 695     def getUserMapFromPerforceServer(self):
 696         if self.userMapFromPerforceServer:
 697             return
 698         self.users = {}
 699         self.emails = {}
 700
 701         for output in p4CmdList("users"):
 702             if not output.has_key("User"):
 703                 continue
 704             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 705             self.emails[output["Email"]] = output["User"]
 706
 707
 708         s = ''
 709         for (key, val) in self.users.items():
 710             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 711
 712         open(self.getUserCacheFilename(), "wb").write(s)
 713         self.userMapFromPerforceServer = True
 714
 715     def loadUserMapFromCache(self):
 716         self.users = {}
 717         self.userMapFromPerforceServer = False
 718         try:
 719             cache = open(self.getUserCacheFilename(), "rb")
 720             lines = cache.readlines()
 721             cache.close()
 722             for line in lines:
 723                 entry = line.strip().split("\t")
 724                 self.users[entry[0]] = entry[1]
 725         except IOError:
 726             self.getUserMapFromPerforceServer()
 727
 728 class P4Debug(Command):
 729     def __init__(self):
 730         Command.__init__(self)
 731         self.options = []
 732         self.description = "A tool to debug the output of p4 -G."
 733         self.needsGit = False
 734
 735     def run(self, args):
 736         j = 0
 737         for output in p4CmdList(args):
 738             print 'Element: %d' % j
 739             j += 1
 740             print output
 741         return True
 742
 743 class P4RollBack(Command):
 744     def __init__(self):
 745         Command.__init__(self)
 746         self.options = [
 747             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 748         ]
 749         self.description = "A tool to debug the multi-branch import. Don't use :)"
 750         self.rollbackLocalBranches = False
 751
 752     def run(self, args):
 753         if len(args) != 1:
 754             return False
 755         maxChange = int(args[0])
 756
 757         if "p4ExitCode" in p4Cmd("changes -m 1"):
 758             die("Problems executing p4");
 759
 760         if self.rollbackLocalBranches:
 761             refPrefix = "refs/heads/"
 762             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 763         else:
 764             refPrefix = "refs/remotes/"
 765             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 766
 767         for line in lines:
 768             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 769                 line = line.strip()
 770                 ref = refPrefix + line
 771                 log = extractLogMessageFromGitCommit(ref)
 772                 settings = extractSettingsGitLog(log)
 773
 774                 depotPaths = settings['depot-paths']
 775                 change = settings['change']
 776
 777                 changed = False
 778
 779                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 780                                                            for p in depotPaths]))) == 0:
 781                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 782                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 783                     continue
 784
 785                 while change and int(change) > maxChange:
 786                     changed = True
 787                     if self.verbose:
 788                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 789                     system("git update-ref %s \"%s^\"" % (ref, ref))
 790                     log = extractLogMessageFromGitCommit(ref)
 791                     settings =  extractSettingsGitLog(log)
 792
 793
 794                     depotPaths = settings['depot-paths']
 795                     change = settings['change']
 796
 797                 if changed:
 798                     print "%s rewound to %s" % (ref, change)
 799
 800         return True
 801
 802 class P4Submit(Command, P4UserMap):
 803     def __init__(self):
 804         Command.__init__(self)
 805         P4UserMap.__init__(self)
 806         self.options = [
 807                 optparse.make_option("--origin", dest="origin"),
 808                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 809                 # preserve the user, requires relevant p4 permissions
 810                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 811                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 812         ]
 813         self.description = "Submit changes from git to the perforce depot."
 814         self.usage += " [name of git branch to submit into perforce depot]"
 815         self.interactive = True
 816         self.origin = ""
 817         self.detectRenames = False
 818         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 819         self.isWindows = (platform.system() == "Windows")
 820         self.exportLabels = False
 821
 822     def check(self):
 823         if len(p4CmdList("opened ...")) > 0:
 824             die("You have files opened with perforce! Close them before starting the sync.")
 825
 826     # replaces everything between 'Description:' and the next P4 submit template field with the
 827     # commit message
 828     def prepareLogMessage(self, template, message):
 829         result = ""
 830
 831         inDescriptionSection = False
 832
 833         for line in template.split("\n"):
 834             if line.startswith("#"):
 835                 result += line + "\n"
 836                 continue
 837
 838             if inDescriptionSection:
 839                 if line.startswith("Files:") or line.startswith("Jobs:"):
 840                     inDescriptionSection = False
 841                 else:
 842                     continue
 843             else:
 844                 if line.startswith("Description:"):
 845                     inDescriptionSection = True
 846                     line += "\n"
 847                     for messageLine in message.split("\n"):
 848                         line += "\t" + messageLine + "\n"
 849
 850             result += line + "\n"
 851
 852         return result
 853
 854     def patchRCSKeywords(self, file, pattern):
 855         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
 856         (handle, outFileName) = tempfile.mkstemp(dir='.')
 857         try:
 858             outFile = os.fdopen(handle, "w+")
 859             inFile = open(file, "r")
 860             regexp = re.compile(pattern, re.VERBOSE)
 861             for line in inFile.readlines():
 862                 line = regexp.sub(r'$\1$', line)
 863                 outFile.write(line)
 864             inFile.close()
 865             outFile.close()
 866             # Forcibly overwrite the original file
 867             os.unlink(file)
 868             shutil.move(outFileName, file)
 869         except:
 870             # cleanup our temporary file
 871             os.unlink(outFileName)
 872             print "Failed to strip RCS keywords in %s" % file
 873             raise
 874
 875         print "Patched up RCS keywords in %s" % file
 876
 877     def p4UserForCommit(self,id):
 878         # Return the tuple (perforce user,git email) for a given git commit id
 879         self.getUserMapFromPerforceServer()
 880         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 881         gitEmail = gitEmail.strip()
 882         if not self.emails.has_key(gitEmail):
 883             return (None,gitEmail)
 884         else:
 885             return (self.emails[gitEmail],gitEmail)
 886
 887     def checkValidP4Users(self,commits):
 888         # check if any git authors cannot be mapped to p4 users
 889         for id in commits:
 890             (user,email) = self.p4UserForCommit(id)
 891             if not user:
 892                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 893                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 894                     print "%s" % msg
 895                 else:
 896                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 897
 898     def lastP4Changelist(self):
 899         # Get back the last changelist number submitted in this client spec. This
 900         # then gets used to patch up the username in the change. If the same
 901         # client spec is being used by multiple processes then this might go
 902         # wrong.
 903         results = p4CmdList("client -o")        # find the current client
 904         client = None
 905         for r in results:
 906             if r.has_key('Client'):
 907                 client = r['Client']
 908                 break
 909         if not client:
 910             die("could not get client spec")
 911         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 912         for r in results:
 913             if r.has_key('change'):
 914                 return r['change']
 915         die("Could not get changelist number for last submit - cannot patch up user details")
 916
 917     def modifyChangelistUser(self, changelist, newUser):
 918         # fixup the user field of a changelist after it has been submitted.
 919         changes = p4CmdList("change -o %s" % changelist)
 920         if len(changes) != 1:
 921             die("Bad output from p4 change modifying %s to user %s" %
 922                 (changelist, newUser))
 923
 924         c = changes[0]
 925         if c['User'] == newUser: return   # nothing to do
 926         c['User'] = newUser
 927         input = marshal.dumps(c)
 928
 929         result = p4CmdList("change -f -i", stdin=input)
 930         for r in result:
 931             if r.has_key('code'):
 932                 if r['code'] == 'error':
 933                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 934             if r.has_key('data'):
 935                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 936                 return
 937         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 938
 939     def canChangeChangelists(self):
 940         # check to see if we have p4 admin or super-user permissions, either of
 941         # which are required to modify changelists.
 942         results = p4CmdList(["protects", self.depotPath])
 943         for r in results:
 944             if r.has_key('perm'):
 945                 if r['perm'] == 'admin':
 946                     return 1
 947                 if r['perm'] == 'super':
 948                     return 1
 949         return 0
 950
 951     def prepareSubmitTemplate(self):
 952         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 953         template = ""
 954         inFilesSection = False
 955         for line in p4_read_pipe_lines(['change', '-o']):
 956             if line.endswith("\r\n"):
 957                 line = line[:-2] + "\n"
 958             if inFilesSection:
 959                 if line.startswith("\t"):
 960                     # path starts and ends with a tab
 961                     path = line[1:]
 962                     lastTab = path.rfind("\t")
 963                     if lastTab != -1:
 964                         path = path[:lastTab]
 965                         if not p4PathStartsWith(path, self.depotPath):
 966                             continue
 967                 else:
 968                     inFilesSection = False
 969             else:
 970                 if line.startswith("Files:"):
 971                     inFilesSection = True
 972
 973             template += line
 974
 975         return template
 976
 977     def edit_template(self, template_file):
 978         """Invoke the editor to let the user change the submission
 979            message.  Return true if okay to continue with the submit."""
 980
 981         # if configured to skip the editing part, just submit
 982         if gitConfig("git-p4.skipSubmitEdit") == "true":
 983             return True
 984
 985         # look at the modification time, to check later if the user saved
 986         # the file
 987         mtime = os.stat(template_file).st_mtime
 988
 989         # invoke the editor
 990         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
 991             editor = os.environ.get("P4EDITOR")
 992         else:
 993             editor = read_pipe("git var GIT_EDITOR").strip()
 994         system(editor + " " + template_file)
 995
 996         # If the file was not saved, prompt to see if this patch should
 997         # be skipped.  But skip this verification step if configured so.
 998         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 999             return True
1000
1001         # modification time updated means user saved the file
1002         if os.stat(template_file).st_mtime > mtime:
1003             return True
1004
1005         while True:
1006             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1007             if response == 'y':
1008                 return True
1009             if response == 'n':
1010                 return False
1011
1012     def applyCommit(self, id):
1013         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1014
1015         (p4User, gitEmail) = self.p4UserForCommit(id)
1016
1017         if not self.detectRenames:
1018             # If not explicitly set check the config variable
1019             self.detectRenames = gitConfig("git-p4.detectRenames")
1020
1021         if self.detectRenames.lower() == "false" or self.detectRenames == "":
1022             diffOpts = ""
1023         elif self.detectRenames.lower() == "true":
1024             diffOpts = "-M"
1025         else:
1026             diffOpts = "-M%s" % self.detectRenames
1027
1028         detectCopies = gitConfig("git-p4.detectCopies")
1029         if detectCopies.lower() == "true":
1030             diffOpts += " -C"
1031         elif detectCopies != "" and detectCopies.lower() != "false":
1032             diffOpts += " -C%s" % detectCopies
1033
1034         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1035             diffOpts += " --find-copies-harder"
1036
1037         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
1038         filesToAdd = set()
1039         filesToDelete = set()
1040         editedFiles = set()
1041         filesToChangeExecBit = {}
1042
1043         for line in diff:
1044             diff = parseDiffTreeEntry(line)
1045             modifier = diff['status']
1046             path = diff['src']
1047             if modifier == "M":
1048                 p4_edit(path)
1049                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1050                     filesToChangeExecBit[path] = diff['dst_mode']
1051                 editedFiles.add(path)
1052             elif modifier == "A":
1053                 filesToAdd.add(path)
1054                 filesToChangeExecBit[path] = diff['dst_mode']
1055                 if path in filesToDelete:
1056                     filesToDelete.remove(path)
1057             elif modifier == "D":
1058                 filesToDelete.add(path)
1059                 if path in filesToAdd:
1060                     filesToAdd.remove(path)
1061             elif modifier == "C":
1062                 src, dest = diff['src'], diff['dst']
1063                 p4_integrate(src, dest)
1064                 if diff['src_sha1'] != diff['dst_sha1']:
1065                     p4_edit(dest)
1066                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1067                     p4_edit(dest)
1068                     filesToChangeExecBit[dest] = diff['dst_mode']
1069                 os.unlink(dest)
1070                 editedFiles.add(dest)
1071             elif modifier == "R":
1072                 src, dest = diff['src'], diff['dst']
1073                 p4_integrate(src, dest)
1074                 if diff['src_sha1'] != diff['dst_sha1']:
1075                     p4_edit(dest)
1076                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1077                     p4_edit(dest)
1078                     filesToChangeExecBit[dest] = diff['dst_mode']
1079                 os.unlink(dest)
1080                 editedFiles.add(dest)
1081                 filesToDelete.add(src)
1082             else:
1083                 die("unknown modifier %s for %s" % (modifier, path))
1084
1085         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1086         patchcmd = diffcmd + " | git apply "
1087         tryPatchCmd = patchcmd + "--check -"
1088         applyPatchCmd = patchcmd + "--check --apply -"
1089         patch_succeeded = True
1090
1091         if os.system(tryPatchCmd) != 0:
1092             fixed_rcs_keywords = False
1093             patch_succeeded = False
1094             print "Unfortunately applying the change failed!"
1095
1096             # Patch failed, maybe it's just RCS keyword woes. Look through
1097             # the patch to see if that's possible.
1098             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1099                 file = None
1100                 pattern = None
1101                 kwfiles = {}
1102                 for file in editedFiles | filesToDelete:
1103                     # did this file's delta contain RCS keywords?
1104                     pattern = p4_keywords_regexp_for_file(file)
1105
1106                     if pattern:
1107                         # this file is a possibility...look for RCS keywords.
1108                         regexp = re.compile(pattern, re.VERBOSE)
1109                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1110                             if regexp.search(line):
1111                                 if verbose:
1112                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1113                                 kwfiles[file] = pattern
1114                                 break
1115
1116                 for file in kwfiles:
1117                     if verbose:
1118                         print "zapping %s with %s" % (line,pattern)
1119                     self.patchRCSKeywords(file, kwfiles[file])
1120                     fixed_rcs_keywords = True
1121
1122             if fixed_rcs_keywords:
1123                 print "Retrying the patch with RCS keywords cleaned up"
1124                 if os.system(tryPatchCmd) == 0:
1125                     patch_succeeded = True
1126
1127         if not patch_succeeded:
1128             print "What do you want to do?"
1129             response = "x"
1130             while response != "s" and response != "a" and response != "w":
1131                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1132                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1133             if response == "s":
1134                 print "Skipping! Good luck with the next patches..."
1135                 for f in editedFiles:
1136                     p4_revert(f)
1137                 for f in filesToAdd:
1138                     os.remove(f)
1139                 return
1140             elif response == "a":
1141                 os.system(applyPatchCmd)
1142                 if len(filesToAdd) > 0:
1143                     print "You may also want to call p4 add on the following files:"
1144                     print " ".join(filesToAdd)
1145                 if len(filesToDelete):
1146                     print "The following files should be scheduled for deletion with p4 delete:"
1147                     print " ".join(filesToDelete)
1148                 die("Please resolve and submit the conflict manually and "
1149                     + "continue afterwards with git p4 submit --continue")
1150             elif response == "w":
1151                 system(diffcmd + " > patch.txt")
1152                 print "Patch saved to patch.txt in %s !" % self.clientPath
1153                 die("Please resolve and submit the conflict manually and "
1154                     "continue afterwards with git p4 submit --continue")
1155
1156         system(applyPatchCmd)
1157
1158         for f in filesToAdd:
1159             p4_add(f)
1160         for f in filesToDelete:
1161             p4_revert(f)
1162             p4_delete(f)
1163
1164         # Set/clear executable bits
1165         for f in filesToChangeExecBit.keys():
1166             mode = filesToChangeExecBit[f]
1167             setP4ExecBit(f, mode)
1168
1169         logMessage = extractLogMessageFromGitCommit(id)
1170         logMessage = logMessage.strip()
1171
1172         template = self.prepareSubmitTemplate()
1173
1174         if self.interactive:
1175             submitTemplate = self.prepareLogMessage(template, logMessage)
1176
1177             if self.preserveUser:
1178                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1179
1180             if os.environ.has_key("P4DIFF"):
1181                 del(os.environ["P4DIFF"])
1182             diff = ""
1183             for editedFile in editedFiles:
1184                 diff += p4_read_pipe(['diff', '-du', editedFile])
1185
1186             newdiff = ""
1187             for newFile in filesToAdd:
1188                 newdiff += "==== new file ====\n"
1189                 newdiff += "--- /dev/null\n"
1190                 newdiff += "+++ %s\n" % newFile
1191                 f = open(newFile, "r")
1192                 for line in f.readlines():
1193                     newdiff += "+" + line
1194                 f.close()
1195
1196             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1197                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1198                 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1199                 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1200
1201             separatorLine = "######## everything below this line is just the diff #######\n"
1202
1203             (handle, fileName) = tempfile.mkstemp()
1204             tmpFile = os.fdopen(handle, "w+")
1205             if self.isWindows:
1206                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1207                 separatorLine = separatorLine.replace("\n", "\r\n")
1208                 newdiff = newdiff.replace("\n", "\r\n")
1209             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1210             tmpFile.close()
1211
1212             if self.edit_template(fileName):
1213                 # read the edited message and submit
1214                 tmpFile = open(fileName, "rb")
1215                 message = tmpFile.read()
1216                 tmpFile.close()
1217                 submitTemplate = message[:message.index(separatorLine)]
1218                 if self.isWindows:
1219                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1220                 p4_write_pipe(['submit', '-i'], submitTemplate)
1221
1222                 if self.preserveUser:
1223                     if p4User:
1224                         # Get last changelist number. Cannot easily get it from
1225                         # the submit command output as the output is
1226                         # unmarshalled.
1227                         changelist = self.lastP4Changelist()
1228                         self.modifyChangelistUser(changelist, p4User)
1229             else:
1230                 # skip this patch
1231                 print "Submission cancelled, undoing p4 changes."
1232                 for f in editedFiles:
1233                     p4_revert(f)
1234                 for f in filesToAdd:
1235                     p4_revert(f)
1236                     os.remove(f)
1237
1238             os.remove(fileName)
1239         else:
1240             fileName = "submit.txt"
1241             file = open(fileName, "w+")
1242             file.write(self.prepareLogMessage(template, logMessage))
1243             file.close()
1244             print ("Perforce submit template written as %s. "
1245                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1246                    % (fileName, fileName))
1247
1248     # Export git tags as p4 labels. Create a p4 label and then tag
1249     # with that.
1250     def exportGitTags(self, gitTags):
1251         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1252         if len(validLabelRegexp) == 0:
1253             validLabelRegexp = defaultLabelRegexp
1254         m = re.compile(validLabelRegexp)
1255
1256         for name in gitTags:
1257
1258             if not m.match(name):
1259                 if verbose:
1260                     print "tag %s does not match regexp %s" % (name, validTagRegexp)
1261                 continue
1262
1263             # Get the p4 commit this corresponds to
1264             logMessage = extractLogMessageFromGitCommit(name)
1265             values = extractSettingsGitLog(logMessage)
1266
1267             if not values.has_key('change'):
1268                 # a tag pointing to something not sent to p4; ignore
1269                 if verbose:
1270                     print "git tag %s does not give a p4 commit" % name
1271                 continue
1272             else:
1273                 changelist = values['change']
1274
1275             # Get the tag details.
1276             inHeader = True
1277             isAnnotated = False
1278             body = []
1279             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1280                 l = l.strip()
1281                 if inHeader:
1282                     if re.match(r'tag\s+', l):
1283                         isAnnotated = True
1284                     elif re.match(r'\s*$', l):
1285                         inHeader = False
1286                         continue
1287                 else:
1288                     body.append(l)
1289
1290             if not isAnnotated:
1291                 body = ["lightweight tag imported by git p4\n"]
1292
1293             # Create the label - use the same view as the client spec we are using
1294             clientSpec = getClientSpec()
1295
1296             labelTemplate  = "Label: %s\n" % name
1297             labelTemplate += "Description:\n"
1298             for b in body:
1299                 labelTemplate += "\t" + b + "\n"
1300             labelTemplate += "View:\n"
1301             for mapping in clientSpec.mappings:
1302                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1303
1304             p4_write_pipe(["label", "-i"], labelTemplate)
1305
1306             # Use the label
1307             p4_system(["tag", "-l", name] +
1308                       ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1309
1310             if verbose:
1311                 print "created p4 label for tag %s" % name
1312
1313     def run(self, args):
1314         if len(args) == 0:
1315             self.master = currentGitBranch()
1316             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1317                 die("Detecting current git branch failed!")
1318         elif len(args) == 1:
1319             self.master = args[0]
1320             if not branchExists(self.master):
1321                 die("Branch %s does not exist" % self.master)
1322         else:
1323             return False
1324
1325         allowSubmit = gitConfig("git-p4.allowSubmit")
1326         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1327             die("%s is not in git-p4.allowSubmit" % self.master)
1328
1329         [upstream, settings] = findUpstreamBranchPoint()
1330         self.depotPath = settings['depot-paths'][0]
1331         if len(self.origin) == 0:
1332             self.origin = upstream
1333
1334         if self.preserveUser:
1335             if not self.canChangeChangelists():
1336                 die("Cannot preserve user names without p4 super-user or admin permissions")
1337
1338         if self.verbose:
1339             print "Origin branch is " + self.origin
1340
1341         if len(self.depotPath) == 0:
1342             print "Internal error: cannot locate perforce depot path from existing branches"
1343             sys.exit(128)
1344
1345         self.useClientSpec = False
1346         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1347             self.useClientSpec = True
1348         if self.useClientSpec:
1349             self.clientSpecDirs = getClientSpec()
1350
1351         if self.useClientSpec:
1352             # all files are relative to the client spec
1353             self.clientPath = getClientRoot()
1354         else:
1355             self.clientPath = p4Where(self.depotPath)
1356
1357         if self.clientPath == "":
1358             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1359
1360         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1361         self.oldWorkingDirectory = os.getcwd()
1362
1363         # ensure the clientPath exists
1364         new_client_dir = False
1365         if not os.path.exists(self.clientPath):
1366             new_client_dir = True
1367             os.makedirs(self.clientPath)
1368
1369         chdir(self.clientPath)
1370         print "Synchronizing p4 checkout..."
1371         if new_client_dir:
1372             # old one was destroyed, and maybe nobody told p4
1373             p4_sync("...", "-f")
1374         else:
1375             p4_sync("...")
1376         self.check()
1377
1378         commits = []
1379         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1380             commits.append(line.strip())
1381         commits.reverse()
1382
1383         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1384             self.checkAuthorship = False
1385         else:
1386             self.checkAuthorship = True
1387
1388         if self.preserveUser:
1389             self.checkValidP4Users(commits)
1390
1391         while len(commits) > 0:
1392             commit = commits[0]
1393             commits = commits[1:]
1394             self.applyCommit(commit)
1395             if not self.interactive:
1396                 break
1397
1398         if len(commits) == 0:
1399             print "All changes applied!"
1400             chdir(self.oldWorkingDirectory)
1401
1402             sync = P4Sync()
1403             sync.run([])
1404
1405             rebase = P4Rebase()
1406             rebase.rebase()
1407
1408         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1409             self.exportLabels = true
1410
1411         if self.exportLabels:
1412             p4Labels = getP4Labels(self.depotPath)
1413             gitTags = getGitTags()
1414
1415             missingGitTags = gitTags - p4Labels
1416             self.exportGitTags(missingGitTags)
1417
1418         return True
1419
1420 class View(object):
1421     """Represent a p4 view ("p4 help views"), and map files in a
1422        repo according to the view."""
1423
1424     class Path(object):
1425         """A depot or client path, possibly containing wildcards.
1426            The only one supported is ... at the end, currently.
1427            Initialize with the full path, with //depot or //client."""
1428
1429         def __init__(self, path, is_depot):
1430             self.path = path
1431             self.is_depot = is_depot
1432             self.find_wildcards()
1433             # remember the prefix bit, useful for relative mappings
1434             m = re.match("(//[^/]+/)", self.path)
1435             if not m:
1436                 die("Path %s does not start with //prefix/" % self.path)
1437             prefix = m.group(1)
1438             if not self.is_depot:
1439                 # strip //client/ on client paths
1440                 self.path = self.path[len(prefix):]
1441
1442         def find_wildcards(self):
1443             """Make sure wildcards are valid, and set up internal
1444                variables."""
1445
1446             self.ends_triple_dot = False
1447             # There are three wildcards allowed in p4 views
1448             # (see "p4 help views").  This code knows how to
1449             # handle "..." (only at the end), but cannot deal with
1450             # "%%n" or "*".  Only check the depot_side, as p4 should
1451             # validate that the client_side matches too.
1452             if re.search(r'%%[1-9]', self.path):
1453                 die("Can't handle %%n wildcards in view: %s" % self.path)
1454             if self.path.find("*") >= 0:
1455                 die("Can't handle * wildcards in view: %s" % self.path)
1456             triple_dot_index = self.path.find("...")
1457             if triple_dot_index >= 0:
1458                 if triple_dot_index != len(self.path) - 3:
1459                     die("Can handle only single ... wildcard, at end: %s" %
1460                         self.path)
1461                 self.ends_triple_dot = True
1462
1463         def ensure_compatible(self, other_path):
1464             """Make sure the wildcards agree."""
1465             if self.ends_triple_dot != other_path.ends_triple_dot:
1466                  die("Both paths must end with ... if either does;\n" +
1467                      "paths: %s %s" % (self.path, other_path.path))
1468
1469         def match_wildcards(self, test_path):
1470             """See if this test_path matches us, and fill in the value
1471                of the wildcards if so.  Returns a tuple of
1472                (True|False, wildcards[]).  For now, only the ... at end
1473                is supported, so at most one wildcard."""
1474             if self.ends_triple_dot:
1475                 dotless = self.path[:-3]
1476                 if test_path.startswith(dotless):
1477                     wildcard = test_path[len(dotless):]
1478                     return (True, [ wildcard ])
1479             else:
1480                 if test_path == self.path:
1481                     return (True, [])
1482             return (False, [])
1483
1484         def match(self, test_path):
1485             """Just return if it matches; don't bother with the wildcards."""
1486             b, _ = self.match_wildcards(test_path)
1487             return b
1488
1489         def fill_in_wildcards(self, wildcards):
1490             """Return the relative path, with the wildcards filled in
1491                if there are any."""
1492             if self.ends_triple_dot:
1493                 return self.path[:-3] + wildcards[0]
1494             else:
1495                 return self.path
1496
1497     class Mapping(object):
1498         def __init__(self, depot_side, client_side, overlay, exclude):
1499             # depot_side is without the trailing /... if it had one
1500             self.depot_side = View.Path(depot_side, is_depot=True)
1501             self.client_side = View.Path(client_side, is_depot=False)
1502             self.overlay = overlay  # started with "+"
1503             self.exclude = exclude  # started with "-"
1504             assert not (self.overlay and self.exclude)
1505             self.depot_side.ensure_compatible(self.client_side)
1506
1507         def __str__(self):
1508             c = " "
1509             if self.overlay:
1510                 c = "+"
1511             if self.exclude:
1512                 c = "-"
1513             return "View.Mapping: %s%s -> %s" % \
1514                    (c, self.depot_side.path, self.client_side.path)
1515
1516         def map_depot_to_client(self, depot_path):
1517             """Calculate the client path if using this mapping on the
1518                given depot path; does not consider the effect of other
1519                mappings in a view.  Even excluded mappings are returned."""
1520             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1521             if not matches:
1522                 return ""
1523             client_path = self.client_side.fill_in_wildcards(wildcards)
1524             return client_path
1525
1526     #
1527     # View methods
1528     #
1529     def __init__(self):
1530         self.mappings = []
1531
1532     def append(self, view_line):
1533         """Parse a view line, splitting it into depot and client
1534            sides.  Append to self.mappings, preserving order."""
1535
1536         # Split the view line into exactly two words.  P4 enforces
1537         # structure on these lines that simplifies this quite a bit.
1538         #
1539         # Either or both words may be double-quoted.
1540         # Single quotes do not matter.
1541         # Double-quote marks cannot occur inside the words.
1542         # A + or - prefix is also inside the quotes.
1543         # There are no quotes unless they contain a space.
1544         # The line is already white-space stripped.
1545         # The two words are separated by a single space.
1546         #
1547         if view_line[0] == '"':
1548             # First word is double quoted.  Find its end.
1549             close_quote_index = view_line.find('"', 1)
1550             if close_quote_index <= 0:
1551                 die("No first-word closing quote found: %s" % view_line)
1552             depot_side = view_line[1:close_quote_index]
1553             # skip closing quote and space
1554             rhs_index = close_quote_index + 1 + 1
1555         else:
1556             space_index = view_line.find(" ")
1557             if space_index <= 0:
1558                 die("No word-splitting space found: %s" % view_line)
1559             depot_side = view_line[0:space_index]
1560             rhs_index = space_index + 1
1561
1562         if view_line[rhs_index] == '"':
1563             # Second word is double quoted.  Make sure there is a
1564             # double quote at the end too.
1565             if not view_line.endswith('"'):
1566                 die("View line with rhs quote should end with one: %s" %
1567                     view_line)
1568             # skip the quotes
1569             client_side = view_line[rhs_index+1:-1]
1570         else:
1571             client_side = view_line[rhs_index:]
1572
1573         # prefix + means overlay on previous mapping
1574         overlay = False
1575         if depot_side.startswith("+"):
1576             overlay = True
1577             depot_side = depot_side[1:]
1578
1579         # prefix - means exclude this path
1580         exclude = False
1581         if depot_side.startswith("-"):
1582             exclude = True
1583             depot_side = depot_side[1:]
1584
1585         m = View.Mapping(depot_side, client_side, overlay, exclude)
1586         self.mappings.append(m)
1587
1588     def map_in_client(self, depot_path):
1589         """Return the relative location in the client where this
1590            depot file should live.  Returns "" if the file should
1591            not be mapped in the client."""
1592
1593         paths_filled = []
1594         client_path = ""
1595
1596         # look at later entries first
1597         for m in self.mappings[::-1]:
1598
1599             # see where will this path end up in the client
1600             p = m.map_depot_to_client(depot_path)
1601
1602             if p == "":
1603                 # Depot path does not belong in client.  Must remember
1604                 # this, as previous items should not cause files to
1605                 # exist in this path either.  Remember that the list is
1606                 # being walked from the end, which has higher precedence.
1607                 # Overlap mappings do not exclude previous mappings.
1608                 if not m.overlay:
1609                     paths_filled.append(m.client_side)
1610
1611             else:
1612                 # This mapping matched; no need to search any further.
1613                 # But, the mapping could be rejected if the client path
1614                 # has already been claimed by an earlier mapping (i.e.
1615                 # one later in the list, which we are walking backwards).
1616                 already_mapped_in_client = False
1617                 for f in paths_filled:
1618                     # this is View.Path.match
1619                     if f.match(p):
1620                         already_mapped_in_client = True
1621                         break
1622                 if not already_mapped_in_client:
1623                     # Include this file, unless it is from a line that
1624                     # explicitly said to exclude it.
1625                     if not m.exclude:
1626                         client_path = p
1627
1628                 # a match, even if rejected, always stops the search
1629                 break
1630
1631         return client_path
1632
1633 class P4Sync(Command, P4UserMap):
1634     delete_actions = ( "delete", "move/delete", "purge" )
1635
1636     def __init__(self):
1637         Command.__init__(self)
1638         P4UserMap.__init__(self)
1639         self.options = [
1640                 optparse.make_option("--branch", dest="branch"),
1641                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1642                 optparse.make_option("--changesfile", dest="changesFile"),
1643                 optparse.make_option("--silent", dest="silent", action="store_true"),
1644                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1645                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1646                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1647                                      help="Import into refs/heads/ , not refs/remotes"),
1648                 optparse.make_option("--max-changes", dest="maxChanges"),
1649                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1650                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1651                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1652                                      help="Only sync files that are included in the Perforce Client Spec")
1653         ]
1654         self.description = """Imports from Perforce into a git repository.\n
1655     example:
1656     //depot/my/project/ -- to import the current head
1657     //depot/my/project/@all -- to import everything
1658     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1659
1660     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1661
1662         self.usage += " //depot/path[@revRange]"
1663         self.silent = False
1664         self.createdBranches = set()
1665         self.committedChanges = set()
1666         self.branch = ""
1667         self.detectBranches = False
1668         self.detectLabels = False
1669         self.importLabels = False
1670         self.changesFile = ""
1671         self.syncWithOrigin = True
1672         self.importIntoRemotes = True
1673         self.maxChanges = ""
1674         self.isWindows = (platform.system() == "Windows")
1675         self.keepRepoPath = False
1676         self.depotPaths = None
1677         self.p4BranchesInGit = []
1678         self.cloneExclude = []
1679         self.useClientSpec = False
1680         self.useClientSpec_from_options = False
1681         self.clientSpecDirs = None
1682         self.tempBranches = []
1683         self.tempBranchLocation = "git-p4-tmp"
1684
1685         if gitConfig("git-p4.syncFromOrigin") == "false":
1686             self.syncWithOrigin = False
1687
1688     #
1689     # P4 wildcards are not allowed in filenames.  P4 complains
1690     # if you simply add them, but you can force it with "-f", in
1691     # which case it translates them into %xx encoding internally.
1692     # Search for and fix just these four characters.  Do % last so
1693     # that fixing it does not inadvertently create new %-escapes.
1694     #
1695     def wildcard_decode(self, path):
1696         # Cannot have * in a filename in windows; untested as to
1697         # what p4 would do in such a case.
1698         if not self.isWindows:
1699             path = path.replace("%2A", "*")
1700         path = path.replace("%23", "#") \
1701                    .replace("%40", "@") \
1702                    .replace("%25", "%")
1703         return path
1704
1705     # Force a checkpoint in fast-import and wait for it to finish
1706     def checkpoint(self):
1707         self.gitStream.write("checkpoint\n\n")
1708         self.gitStream.write("progress checkpoint\n\n")
1709         out = self.gitOutput.readline()
1710         if self.verbose:
1711             print "checkpoint finished: " + out
1712
1713     def extractFilesFromCommit(self, commit):
1714         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1715                              for path in self.cloneExclude]
1716         files = []
1717         fnum = 0
1718         while commit.has_key("depotFile%s" % fnum):
1719             path =  commit["depotFile%s" % fnum]
1720
1721             if [p for p in self.cloneExclude
1722                 if p4PathStartsWith(path, p)]:
1723                 found = False
1724             else:
1725                 found = [p for p in self.depotPaths
1726                          if p4PathStartsWith(path, p)]
1727             if not found:
1728                 fnum = fnum + 1
1729                 continue
1730
1731             file = {}
1732             file["path"] = path
1733             file["rev"] = commit["rev%s" % fnum]
1734             file["action"] = commit["action%s" % fnum]
1735             file["type"] = commit["type%s" % fnum]
1736             files.append(file)
1737             fnum = fnum + 1
1738         return files
1739
1740     def stripRepoPath(self, path, prefixes):
1741         if self.useClientSpec:
1742             return self.clientSpecDirs.map_in_client(path)
1743
1744         if self.keepRepoPath:
1745             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1746
1747         for p in prefixes:
1748             if p4PathStartsWith(path, p):
1749                 path = path[len(p):]
1750
1751         return path
1752
1753     def splitFilesIntoBranches(self, commit):
1754         branches = {}
1755         fnum = 0
1756         while commit.has_key("depotFile%s" % fnum):
1757             path =  commit["depotFile%s" % fnum]
1758             found = [p for p in self.depotPaths
1759                      if p4PathStartsWith(path, p)]
1760             if not found:
1761                 fnum = fnum + 1
1762                 continue
1763
1764             file = {}
1765             file["path"] = path
1766             file["rev"] = commit["rev%s" % fnum]
1767             file["action"] = commit["action%s" % fnum]
1768             file["type"] = commit["type%s" % fnum]
1769             fnum = fnum + 1
1770
1771             relPath = self.stripRepoPath(path, self.depotPaths)
1772
1773             for branch in self.knownBranches.keys():
1774
1775                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1776                 if relPath.startswith(branch + "/"):
1777                     if branch not in branches:
1778                         branches[branch] = []
1779                     branches[branch].append(file)
1780                     break
1781
1782         return branches
1783
1784     # output one file from the P4 stream
1785     # - helper for streamP4Files
1786
1787     def streamOneP4File(self, file, contents):
1788         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1789         relPath = self.wildcard_decode(relPath)
1790         if verbose:
1791             sys.stderr.write("%s\n" % relPath)
1792
1793         (type_base, type_mods) = split_p4_type(file["type"])
1794
1795         git_mode = "100644"
1796         if "x" in type_mods:
1797             git_mode = "100755"
1798         if type_base == "symlink":
1799             git_mode = "120000"
1800             # p4 print on a symlink contains "target\n"; remove the newline
1801             data = ''.join(contents)
1802             contents = [data[:-1]]
1803
1804         if type_base == "utf16":
1805             # p4 delivers different text in the python output to -G
1806             # than it does when using "print -o", or normal p4 client
1807             # operations.  utf16 is converted to ascii or utf8, perhaps.
1808             # But ascii text saved as -t utf16 is completely mangled.
1809             # Invoke print -o to get the real contents.
1810             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1811             contents = [ text ]
1812
1813         if type_base == "apple":
1814             # Apple filetype files will be streamed as a concatenation of
1815             # its appledouble header and the contents.  This is useless
1816             # on both macs and non-macs.  If using "print -q -o xx", it
1817             # will create "xx" with the data, and "%xx" with the header.
1818             # This is also not very useful.
1819             #
1820             # Ideally, someday, this script can learn how to generate
1821             # appledouble files directly and import those to git, but
1822             # non-mac machines can never find a use for apple filetype.
1823             print "\nIgnoring apple filetype file %s" % file['depotFile']
1824             return
1825
1826         # Perhaps windows wants unicode, utf16 newlines translated too;
1827         # but this is not doing it.
1828         if self.isWindows and type_base == "text":
1829             mangled = []
1830             for data in contents:
1831                 data = data.replace("\r\n", "\n")
1832                 mangled.append(data)
1833             contents = mangled
1834
1835         # Note that we do not try to de-mangle keywords on utf16 files,
1836         # even though in theory somebody may want that.
1837         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1838         if pattern:
1839             regexp = re.compile(pattern, re.VERBOSE)
1840             text = ''.join(contents)
1841             text = regexp.sub(r'$\1$', text)
1842             contents = [ text ]
1843
1844         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1845
1846         # total length...
1847         length = 0
1848         for d in contents:
1849             length = length + len(d)
1850
1851         self.gitStream.write("data %d\n" % length)
1852         for d in contents:
1853             self.gitStream.write(d)
1854         self.gitStream.write("\n")
1855
1856     def streamOneP4Deletion(self, file):
1857         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1858         if verbose:
1859             sys.stderr.write("delete %s\n" % relPath)
1860         self.gitStream.write("D %s\n" % relPath)
1861
1862     # handle another chunk of streaming data
1863     def streamP4FilesCb(self, marshalled):
1864
1865         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1866             # start of a new file - output the old one first
1867             self.streamOneP4File(self.stream_file, self.stream_contents)
1868             self.stream_file = {}
1869             self.stream_contents = []
1870             self.stream_have_file_info = False
1871
1872         # pick up the new file information... for the
1873         # 'data' field we need to append to our array
1874         for k in marshalled.keys():
1875             if k == 'data':
1876                 self.stream_contents.append(marshalled['data'])
1877             else:
1878                 self.stream_file[k] = marshalled[k]
1879
1880         self.stream_have_file_info = True
1881
1882     # Stream directly from "p4 files" into "git fast-import"
1883     def streamP4Files(self, files):
1884         filesForCommit = []
1885         filesToRead = []
1886         filesToDelete = []
1887
1888         for f in files:
1889             # if using a client spec, only add the files that have
1890             # a path in the client
1891             if self.clientSpecDirs:
1892                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1893                     continue
1894
1895             filesForCommit.append(f)
1896             if f['action'] in self.delete_actions:
1897                 filesToDelete.append(f)
1898             else:
1899                 filesToRead.append(f)
1900
1901         # deleted files...
1902         for f in filesToDelete:
1903             self.streamOneP4Deletion(f)
1904
1905         if len(filesToRead) > 0:
1906             self.stream_file = {}
1907             self.stream_contents = []
1908             self.stream_have_file_info = False
1909
1910             # curry self argument
1911             def streamP4FilesCbSelf(entry):
1912                 self.streamP4FilesCb(entry)
1913
1914             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1915
1916             p4CmdList(["-x", "-", "print"],
1917                       stdin=fileArgs,
1918                       cb=streamP4FilesCbSelf)
1919
1920             # do the last chunk
1921             if self.stream_file.has_key('depotFile'):
1922                 self.streamOneP4File(self.stream_file, self.stream_contents)
1923
1924     def make_email(self, userid):
1925         if userid in self.users:
1926             return self.users[userid]
1927         else:
1928             return "%s <a@b>" % userid
1929
1930     # Stream a p4 tag
1931     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
1932         if verbose:
1933             print "writing tag %s for commit %s" % (labelName, commit)
1934         gitStream.write("tag %s\n" % labelName)
1935         gitStream.write("from %s\n" % commit)
1936
1937         if labelDetails.has_key('Owner'):
1938             owner = labelDetails["Owner"]
1939         else:
1940             owner = None
1941
1942         # Try to use the owner of the p4 label, or failing that,
1943         # the current p4 user id.
1944         if owner:
1945             email = self.make_email(owner)
1946         else:
1947             email = self.make_email(self.p4UserId())
1948         tagger = "%s %s %s" % (email, epoch, self.tz)
1949
1950         gitStream.write("tagger %s\n" % tagger)
1951
1952         print "labelDetails=",labelDetails
1953         if labelDetails.has_key('Description'):
1954             description = labelDetails['Description']
1955         else:
1956             description = 'Label from git p4'
1957
1958         gitStream.write("data %d\n" % len(description))
1959         gitStream.write(description)
1960         gitStream.write("\n")
1961
1962     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1963         epoch = details["time"]
1964         author = details["user"]
1965         self.branchPrefixes = branchPrefixes
1966
1967         if self.verbose:
1968             print "commit into %s" % branch
1969
1970         # start with reading files; if that fails, we should not
1971         # create a commit.
1972         new_files = []
1973         for f in files:
1974             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1975                 new_files.append (f)
1976             else:
1977                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1978
1979         self.gitStream.write("commit %s\n" % branch)
1980 #        gitStream.write("mark :%s\n" % details["change"])
1981         self.committedChanges.add(int(details["change"]))
1982         committer = ""
1983         if author not in self.users:
1984             self.getUserMapFromPerforceServer()
1985         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
1986
1987         self.gitStream.write("committer %s\n" % committer)
1988
1989         self.gitStream.write("data <<EOT\n")
1990         self.gitStream.write(details["desc"])
1991         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1992                              % (','.join (branchPrefixes), details["change"]))
1993         if len(details['options']) > 0:
1994             self.gitStream.write(": options = %s" % details['options'])
1995         self.gitStream.write("]\nEOT\n\n")
1996
1997         if len(parent) > 0:
1998             if self.verbose:
1999                 print "parent %s" % parent
2000             self.gitStream.write("from %s\n" % parent)
2001
2002         self.streamP4Files(new_files)
2003         self.gitStream.write("\n")
2004
2005         change = int(details["change"])
2006
2007         if self.labels.has_key(change):
2008             label = self.labels[change]
2009             labelDetails = label[0]
2010             labelRevisions = label[1]
2011             if self.verbose:
2012                 print "Change %s is labelled %s" % (change, labelDetails)
2013
2014             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2015                                                     for p in branchPrefixes])
2016
2017             if len(files) == len(labelRevisions):
2018
2019                 cleanedFiles = {}
2020                 for info in files:
2021                     if info["action"] in self.delete_actions:
2022                         continue
2023                     cleanedFiles[info["depotFile"]] = info["rev"]
2024
2025                 if cleanedFiles == labelRevisions:
2026                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2027
2028                 else:
2029                     if not self.silent:
2030                         print ("Tag %s does not match with change %s: files do not match."
2031                                % (labelDetails["label"], change))
2032
2033             else:
2034                 if not self.silent:
2035                     print ("Tag %s does not match with change %s: file count is different."
2036                            % (labelDetails["label"], change))
2037
2038     # Build a dictionary of changelists and labels, for "detect-labels" option.
2039     def getLabels(self):
2040         self.labels = {}
2041
2042         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2043         if len(l) > 0 and not self.silent:
2044             print "Finding files belonging to labels in %s" % `self.depotPaths`
2045
2046         for output in l:
2047             label = output["label"]
2048             revisions = {}
2049             newestChange = 0
2050             if self.verbose:
2051                 print "Querying files for label %s" % label
2052             for file in p4CmdList(["files"] +
2053                                       ["%s...@%s" % (p, label)
2054                                           for p in self.depotPaths]):
2055                 revisions[file["depotFile"]] = file["rev"]
2056                 change = int(file["change"])
2057                 if change > newestChange:
2058                     newestChange = change
2059
2060             self.labels[newestChange] = [output, revisions]
2061
2062         if self.verbose:
2063             print "Label changes: %s" % self.labels.keys()
2064
2065     # Import p4 labels as git tags. A direct mapping does not
2066     # exist, so assume that if all the files are at the same revision
2067     # then we can use that, or it's something more complicated we should
2068     # just ignore.
2069     def importP4Labels(self, stream, p4Labels):
2070         if verbose:
2071             print "import p4 labels: " + ' '.join(p4Labels)
2072
2073         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2074         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2075         if len(validLabelRegexp) == 0:
2076             validLabelRegexp = defaultLabelRegexp
2077         m = re.compile(validLabelRegexp)
2078
2079         for name in p4Labels:
2080             commitFound = False
2081
2082             if not m.match(name):
2083                 if verbose:
2084                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2085                 continue
2086
2087             if name in ignoredP4Labels:
2088                 continue
2089
2090             labelDetails = p4CmdList(['label', "-o", name])[0]
2091
2092             # get the most recent changelist for each file in this label
2093             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2094                                 for p in self.depotPaths])
2095
2096             if change.has_key('change'):
2097                 # find the corresponding git commit; take the oldest commit
2098                 changelist = int(change['change'])
2099                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2100                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2101                 if len(gitCommit) == 0:
2102                     print "could not find git commit for changelist %d" % changelist
2103                 else:
2104                     gitCommit = gitCommit.strip()
2105                     commitFound = True
2106                     # Convert from p4 time format
2107                     try:
2108                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2109                     except ValueError:
2110                         print "Could not convert label time %s" % labelDetail['Update']
2111                         tmwhen = 1
2112
2113                     when = int(time.mktime(tmwhen))
2114                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2115                     if verbose:
2116                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2117             else:
2118                 if verbose:
2119                     print "Label %s has no changelists - possibly deleted?" % name
2120
2121             if not commitFound:
2122                 # We can't import this label; don't try again as it will get very
2123                 # expensive repeatedly fetching all the files for labels that will
2124                 # never be imported. If the label is moved in the future, the
2125                 # ignore will need to be removed manually.
2126                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2127
2128     def guessProjectName(self):
2129         for p in self.depotPaths:
2130             if p.endswith("/"):
2131                 p = p[:-1]
2132             p = p[p.strip().rfind("/") + 1:]
2133             if not p.endswith("/"):
2134                p += "/"
2135             return p
2136
2137     def getBranchMapping(self):
2138         lostAndFoundBranches = set()
2139
2140         user = gitConfig("git-p4.branchUser")
2141         if len(user) > 0:
2142             command = "branches -u %s" % user
2143         else:
2144             command = "branches"
2145
2146         for info in p4CmdList(command):
2147             details = p4Cmd(["branch", "-o", info["branch"]])
2148             viewIdx = 0
2149             while details.has_key("View%s" % viewIdx):
2150                 paths = details["View%s" % viewIdx].split(" ")
2151                 viewIdx = viewIdx + 1
2152                 # require standard //depot/foo/... //depot/bar/... mapping
2153                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2154                     continue
2155                 source = paths[0]
2156                 destination = paths[1]
2157                 ## HACK
2158                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2159                     source = source[len(self.depotPaths[0]):-4]
2160                     destination = destination[len(self.depotPaths[0]):-4]
2161
2162                     if destination in self.knownBranches:
2163                         if not self.silent:
2164                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2165                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2166                         continue
2167
2168                     self.knownBranches[destination] = source
2169
2170                     lostAndFoundBranches.discard(destination)
2171
2172                     if source not in self.knownBranches:
2173                         lostAndFoundBranches.add(source)
2174
2175         # Perforce does not strictly require branches to be defined, so we also
2176         # check git config for a branch list.
2177         #
2178         # Example of branch definition in git config file:
2179         # [git-p4]
2180         #   branchList=main:branchA
2181         #   branchList=main:branchB
2182         #   branchList=branchA:branchC
2183         configBranches = gitConfigList("git-p4.branchList")
2184         for branch in configBranches:
2185             if branch:
2186                 (source, destination) = branch.split(":")
2187                 self.knownBranches[destination] = source
2188
2189                 lostAndFoundBranches.discard(destination)
2190
2191                 if source not in self.knownBranches:
2192                     lostAndFoundBranches.add(source)
2193
2194
2195         for branch in lostAndFoundBranches:
2196             self.knownBranches[branch] = branch
2197
2198     def getBranchMappingFromGitBranches(self):
2199         branches = p4BranchesInGit(self.importIntoRemotes)
2200         for branch in branches.keys():
2201             if branch == "master":
2202                 branch = "main"
2203             else:
2204                 branch = branch[len(self.projectName):]
2205             self.knownBranches[branch] = branch
2206
2207     def listExistingP4GitBranches(self):
2208         # branches holds mapping from name to commit
2209         branches = p4BranchesInGit(self.importIntoRemotes)
2210         self.p4BranchesInGit = branches.keys()
2211         for branch in branches.keys():
2212             self.initialParents[self.refPrefix + branch] = branches[branch]
2213
2214     def updateOptionDict(self, d):
2215         option_keys = {}
2216         if self.keepRepoPath:
2217             option_keys['keepRepoPath'] = 1
2218
2219         d["options"] = ' '.join(sorted(option_keys.keys()))
2220
2221     def readOptions(self, d):
2222         self.keepRepoPath = (d.has_key('options')
2223                              and ('keepRepoPath' in d['options']))
2224
2225     def gitRefForBranch(self, branch):
2226         if branch == "main":
2227             return self.refPrefix + "master"
2228
2229         if len(branch) <= 0:
2230             return branch
2231
2232         return self.refPrefix + self.projectName + branch
2233
2234     def gitCommitByP4Change(self, ref, change):
2235         if self.verbose:
2236             print "looking in ref " + ref + " for change %s using bisect..." % change
2237
2238         earliestCommit = ""
2239         latestCommit = parseRevision(ref)
2240
2241         while True:
2242             if self.verbose:
2243                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2244             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2245             if len(next) == 0:
2246                 if self.verbose:
2247                     print "argh"
2248                 return ""
2249             log = extractLogMessageFromGitCommit(next)
2250             settings = extractSettingsGitLog(log)
2251             currentChange = int(settings['change'])
2252             if self.verbose:
2253                 print "current change %s" % currentChange
2254
2255             if currentChange == change:
2256                 if self.verbose:
2257                     print "found %s" % next
2258                 return next
2259
2260             if currentChange < change:
2261                 earliestCommit = "^%s" % next
2262             else:
2263                 latestCommit = "%s" % next
2264
2265         return ""
2266
2267     def importNewBranch(self, branch, maxChange):
2268         # make fast-import flush all changes to disk and update the refs using the checkpoint
2269         # command so that we can try to find the branch parent in the git history
2270         self.gitStream.write("checkpoint\n\n");
2271         self.gitStream.flush();
2272         branchPrefix = self.depotPaths[0] + branch + "/"
2273         range = "@1,%s" % maxChange
2274         #print "prefix" + branchPrefix
2275         changes = p4ChangesForPaths([branchPrefix], range)
2276         if len(changes) <= 0:
2277             return False
2278         firstChange = changes[0]
2279         #print "first change in branch: %s" % firstChange
2280         sourceBranch = self.knownBranches[branch]
2281         sourceDepotPath = self.depotPaths[0] + sourceBranch
2282         sourceRef = self.gitRefForBranch(sourceBranch)
2283         #print "source " + sourceBranch
2284
2285         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2286         #print "branch parent: %s" % branchParentChange
2287         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2288         if len(gitParent) > 0:
2289             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2290             #print "parent git commit: %s" % gitParent
2291
2292         self.importChanges(changes)
2293         return True
2294
2295     def searchParent(self, parent, branch, target):
2296         parentFound = False
2297         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2298             blob = blob.strip()
2299             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2300                 parentFound = True
2301                 if self.verbose:
2302                     print "Found parent of %s in commit %s" % (branch, blob)
2303                 break
2304         if parentFound:
2305             return blob
2306         else:
2307             return None
2308
2309     def importChanges(self, changes):
2310         cnt = 1
2311         for change in changes:
2312             description = p4Cmd(["describe", str(change)])
2313             self.updateOptionDict(description)
2314
2315             if not self.silent:
2316                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2317                 sys.stdout.flush()
2318             cnt = cnt + 1
2319
2320             try:
2321                 if self.detectBranches:
2322                     branches = self.splitFilesIntoBranches(description)
2323                     for branch in branches.keys():
2324                         ## HACK  --hwn
2325                         branchPrefix = self.depotPaths[0] + branch + "/"
2326
2327                         parent = ""
2328
2329                         filesForCommit = branches[branch]
2330
2331                         if self.verbose:
2332                             print "branch is %s" % branch
2333
2334                         self.updatedBranches.add(branch)
2335
2336                         if branch not in self.createdBranches:
2337                             self.createdBranches.add(branch)
2338                             parent = self.knownBranches[branch]
2339                             if parent == branch:
2340                                 parent = ""
2341                             else:
2342                                 fullBranch = self.projectName + branch
2343                                 if fullBranch not in self.p4BranchesInGit:
2344                                     if not self.silent:
2345                                         print("\n    Importing new branch %s" % fullBranch);
2346                                     if self.importNewBranch(branch, change - 1):
2347                                         parent = ""
2348                                         self.p4BranchesInGit.append(fullBranch)
2349                                     if not self.silent:
2350                                         print("\n    Resuming with change %s" % change);
2351
2352                                 if self.verbose:
2353                                     print "parent determined through known branches: %s" % parent
2354
2355                         branch = self.gitRefForBranch(branch)
2356                         parent = self.gitRefForBranch(parent)
2357
2358                         if self.verbose:
2359                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2360
2361                         if len(parent) == 0 and branch in self.initialParents:
2362                             parent = self.initialParents[branch]
2363                             del self.initialParents[branch]
2364
2365                         blob = None
2366                         if len(parent) > 0:
2367                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2368                             if self.verbose:
2369                                 print "Creating temporary branch: " + tempBranch
2370                             self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2371                             self.tempBranches.append(tempBranch)
2372                             self.checkpoint()
2373                             blob = self.searchParent(parent, branch, tempBranch)
2374                         if blob:
2375                             self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2376                         else:
2377                             if self.verbose:
2378                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2379                             self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2380                 else:
2381                     files = self.extractFilesFromCommit(description)
2382                     self.commit(description, files, self.branch, self.depotPaths,
2383                                 self.initialParent)
2384                     self.initialParent = ""
2385             except IOError:
2386                 print self.gitError.read()
2387                 sys.exit(1)
2388
2389     def importHeadRevision(self, revision):
2390         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2391
2392         details = {}
2393         details["user"] = "git perforce import user"
2394         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2395                            % (' '.join(self.depotPaths), revision))
2396         details["change"] = revision
2397         newestRevision = 0
2398
2399         fileCnt = 0
2400         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2401
2402         for info in p4CmdList(["files"] + fileArgs):
2403
2404             if 'code' in info and info['code'] == 'error':
2405                 sys.stderr.write("p4 returned an error: %s\n"
2406                                  % info['data'])
2407                 if info['data'].find("must refer to client") >= 0:
2408                     sys.stderr.write("This particular p4 error is misleading.\n")
2409                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2410                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2411                 sys.exit(1)
2412             if 'p4ExitCode' in info:
2413                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2414                 sys.exit(1)
2415
2416
2417             change = int(info["change"])
2418             if change > newestRevision:
2419                 newestRevision = change
2420
2421             if info["action"] in self.delete_actions:
2422                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2423                 #fileCnt = fileCnt + 1
2424                 continue
2425
2426             for prop in ["depotFile", "rev", "action", "type" ]:
2427                 details["%s%s" % (prop, fileCnt)] = info[prop]
2428
2429             fileCnt = fileCnt + 1
2430
2431         details["change"] = newestRevision
2432
2433         # Use time from top-most change so that all git p4 clones of
2434         # the same p4 repo have the same commit SHA1s.
2435         res = p4CmdList("describe -s %d" % newestRevision)
2436         newestTime = None
2437         for r in res:
2438             if r.has_key('time'):
2439                 newestTime = int(r['time'])
2440         if newestTime is None:
2441             die("\"describe -s\" on newest change %d did not give a time")
2442         details["time"] = newestTime
2443
2444         self.updateOptionDict(details)
2445         try:
2446             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2447         except IOError:
2448             print "IO error with git fast-import. Is your git version recent enough?"
2449             print self.gitError.read()
2450
2451
2452     def run(self, args):
2453         self.depotPaths = []
2454         self.changeRange = ""
2455         self.initialParent = ""
2456         self.previousDepotPaths = []
2457
2458         # map from branch depot path to parent branch
2459         self.knownBranches = {}
2460         self.initialParents = {}
2461         self.hasOrigin = originP4BranchesExist()
2462         if not self.syncWithOrigin:
2463             self.hasOrigin = False
2464
2465         if self.importIntoRemotes:
2466             self.refPrefix = "refs/remotes/p4/"
2467         else:
2468             self.refPrefix = "refs/heads/p4/"
2469
2470         if self.syncWithOrigin and self.hasOrigin:
2471             if not self.silent:
2472                 print "Syncing with origin first by calling git fetch origin"
2473             system("git fetch origin")
2474
2475         if len(self.branch) == 0:
2476             self.branch = self.refPrefix + "master"
2477             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2478                 system("git update-ref %s refs/heads/p4" % self.branch)
2479                 system("git branch -D p4");
2480             # create it /after/ importing, when master exists
2481             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2482                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2483
2484         # accept either the command-line option, or the configuration variable
2485         if self.useClientSpec:
2486             # will use this after clone to set the variable
2487             self.useClientSpec_from_options = True
2488         else:
2489             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2490                 self.useClientSpec = True
2491         if self.useClientSpec:
2492             self.clientSpecDirs = getClientSpec()
2493
2494         # TODO: should always look at previous commits,
2495         # merge with previous imports, if possible.
2496         if args == []:
2497             if self.hasOrigin:
2498                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2499             self.listExistingP4GitBranches()
2500
2501             if len(self.p4BranchesInGit) > 1:
2502                 if not self.silent:
2503                     print "Importing from/into multiple branches"
2504                 self.detectBranches = True
2505
2506             if self.verbose:
2507                 print "branches: %s" % self.p4BranchesInGit
2508
2509             p4Change = 0
2510             for branch in self.p4BranchesInGit:
2511                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2512
2513                 settings = extractSettingsGitLog(logMsg)
2514
2515                 self.readOptions(settings)
2516                 if (settings.has_key('depot-paths')
2517                     and settings.has_key ('change')):
2518                     change = int(settings['change']) + 1
2519                     p4Change = max(p4Change, change)
2520
2521                     depotPaths = sorted(settings['depot-paths'])
2522                     if self.previousDepotPaths == []:
2523                         self.previousDepotPaths = depotPaths
2524                     else:
2525                         paths = []
2526                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2527                             prev_list = prev.split("/")
2528                             cur_list = cur.split("/")
2529                             for i in range(0, min(len(cur_list), len(prev_list))):
2530                                 if cur_list[i] <> prev_list[i]:
2531                                     i = i - 1
2532                                     break
2533
2534                             paths.append ("/".join(cur_list[:i + 1]))
2535
2536                         self.previousDepotPaths = paths
2537
2538             if p4Change > 0:
2539                 self.depotPaths = sorted(self.previousDepotPaths)
2540                 self.changeRange = "@%s,#head" % p4Change
2541                 if not self.detectBranches:
2542                     self.initialParent = parseRevision(self.branch)
2543                 if not self.silent and not self.detectBranches:
2544                     print "Performing incremental import into %s git branch" % self.branch
2545
2546         if not self.branch.startswith("refs/"):
2547             self.branch = "refs/heads/" + self.branch
2548
2549         if len(args) == 0 and self.depotPaths:
2550             if not self.silent:
2551                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2552         else:
2553             if self.depotPaths and self.depotPaths != args:
2554                 print ("previous import used depot path %s and now %s was specified. "
2555                        "This doesn't work!" % (' '.join (self.depotPaths),
2556                                                ' '.join (args)))
2557                 sys.exit(1)
2558
2559             self.depotPaths = sorted(args)
2560
2561         revision = ""
2562         self.users = {}
2563
2564         # Make sure no revision specifiers are used when --changesfile
2565         # is specified.
2566         bad_changesfile = False
2567         if len(self.changesFile) > 0:
2568             for p in self.depotPaths:
2569                 if p.find("@") >= 0 or p.find("#") >= 0:
2570                     bad_changesfile = True
2571                     break
2572         if bad_changesfile:
2573             die("Option --changesfile is incompatible with revision specifiers")
2574
2575         newPaths = []
2576         for p in self.depotPaths:
2577             if p.find("@") != -1:
2578                 atIdx = p.index("@")
2579                 self.changeRange = p[atIdx:]
2580                 if self.changeRange == "@all":
2581                     self.changeRange = ""
2582                 elif ',' not in self.changeRange:
2583                     revision = self.changeRange
2584                     self.changeRange = ""
2585                 p = p[:atIdx]
2586             elif p.find("#") != -1:
2587                 hashIdx = p.index("#")
2588                 revision = p[hashIdx:]
2589                 p = p[:hashIdx]
2590             elif self.previousDepotPaths == []:
2591                 # pay attention to changesfile, if given, else import
2592                 # the entire p4 tree at the head revision
2593                 if len(self.changesFile) == 0:
2594                     revision = "#head"
2595
2596             p = re.sub ("\.\.\.$", "", p)
2597             if not p.endswith("/"):
2598                 p += "/"
2599
2600             newPaths.append(p)
2601
2602         self.depotPaths = newPaths
2603
2604         self.loadUserMapFromCache()
2605         self.labels = {}
2606         if self.detectLabels:
2607             self.getLabels();
2608
2609         if self.detectBranches:
2610             ## FIXME - what's a P4 projectName ?
2611             self.projectName = self.guessProjectName()
2612
2613             if self.hasOrigin:
2614                 self.getBranchMappingFromGitBranches()
2615             else:
2616                 self.getBranchMapping()
2617             if self.verbose:
2618                 print "p4-git branches: %s" % self.p4BranchesInGit
2619                 print "initial parents: %s" % self.initialParents
2620             for b in self.p4BranchesInGit:
2621                 if b != "master":
2622
2623                     ## FIXME
2624                     b = b[len(self.projectName):]
2625                 self.createdBranches.add(b)
2626
2627         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2628
2629         importProcess = subprocess.Popen(["git", "fast-import"],
2630                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2631                                          stderr=subprocess.PIPE);
2632         self.gitOutput = importProcess.stdout
2633         self.gitStream = importProcess.stdin
2634         self.gitError = importProcess.stderr
2635
2636         if revision:
2637             self.importHeadRevision(revision)
2638         else:
2639             changes = []
2640
2641             if len(self.changesFile) > 0:
2642                 output = open(self.changesFile).readlines()
2643                 changeSet = set()
2644                 for line in output:
2645                     changeSet.add(int(line))
2646
2647                 for change in changeSet:
2648                     changes.append(change)
2649
2650                 changes.sort()
2651             else:
2652                 # catch "git p4 sync" with no new branches, in a repo that
2653                 # does not have any existing p4 branches
2654                 if len(args) == 0 and not self.p4BranchesInGit:
2655                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2656                 if self.verbose:
2657                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2658                                                               self.changeRange)
2659                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2660
2661                 if len(self.maxChanges) > 0:
2662                     changes = changes[:min(int(self.maxChanges), len(changes))]
2663
2664             if len(changes) == 0:
2665                 if not self.silent:
2666                     print "No changes to import!"
2667             else:
2668                 if not self.silent and not self.detectBranches:
2669                     print "Import destination: %s" % self.branch
2670
2671                 self.updatedBranches = set()
2672
2673                 self.importChanges(changes)
2674
2675                 if not self.silent:
2676                     print ""
2677                     if len(self.updatedBranches) > 0:
2678                         sys.stdout.write("Updated branches: ")
2679                         for b in self.updatedBranches:
2680                             sys.stdout.write("%s " % b)
2681                         sys.stdout.write("\n")
2682
2683         if gitConfig("git-p4.importLabels", "--bool") == "true":
2684             self.importLabels = true
2685
2686         if self.importLabels:
2687             p4Labels = getP4Labels(self.depotPaths)
2688             gitTags = getGitTags()
2689
2690             missingP4Labels = p4Labels - gitTags
2691             self.importP4Labels(self.gitStream, missingP4Labels)
2692
2693         self.gitStream.close()
2694         if importProcess.wait() != 0:
2695             die("fast-import failed: %s" % self.gitError.read())
2696         self.gitOutput.close()
2697         self.gitError.close()
2698
2699         # Cleanup temporary branches created during import
2700         if self.tempBranches != []:
2701             for branch in self.tempBranches:
2702                 read_pipe("git update-ref -d %s" % branch)
2703             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2704
2705         return True
2706
2707 class P4Rebase(Command):
2708     def __init__(self):
2709         Command.__init__(self)
2710         self.options = [
2711                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2712         ]
2713         self.importLabels = False
2714         self.description = ("Fetches the latest revision from perforce and "
2715                             + "rebases the current work (branch) against it")
2716
2717     def run(self, args):
2718         sync = P4Sync()
2719         sync.importLabels = self.importLabels
2720         sync.run([])
2721
2722         return self.rebase()
2723
2724     def rebase(self):
2725         if os.system("git update-index --refresh") != 0:
2726             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2727         if len(read_pipe("git diff-index HEAD --")) > 0:
2728             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2729
2730         [upstream, settings] = findUpstreamBranchPoint()
2731         if len(upstream) == 0:
2732             die("Cannot find upstream branchpoint for rebase")
2733
2734         # the branchpoint may be p4/foo~3, so strip off the parent
2735         upstream = re.sub("~[0-9]+$", "", upstream)
2736
2737         print "Rebasing the current branch onto %s" % upstream
2738         oldHead = read_pipe("git rev-parse HEAD").strip()
2739         system("git rebase %s" % upstream)
2740         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2741         return True
2742
2743 class P4Clone(P4Sync):
2744     def __init__(self):
2745         P4Sync.__init__(self)
2746         self.description = "Creates a new git repository and imports from Perforce into it"
2747         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2748         self.options += [
2749             optparse.make_option("--destination", dest="cloneDestination",
2750                                  action='store', default=None,
2751                                  help="where to leave result of the clone"),
2752             optparse.make_option("-/", dest="cloneExclude",
2753                                  action="append", type="string",
2754                                  help="exclude depot path"),
2755             optparse.make_option("--bare", dest="cloneBare",
2756                                  action="store_true", default=False),
2757         ]
2758         self.cloneDestination = None
2759         self.needsGit = False
2760         self.cloneBare = False
2761
2762     # This is required for the "append" cloneExclude action
2763     def ensure_value(self, attr, value):
2764         if not hasattr(self, attr) or getattr(self, attr) is None:
2765             setattr(self, attr, value)
2766         return getattr(self, attr)
2767
2768     def defaultDestination(self, args):
2769         ## TODO: use common prefix of args?
2770         depotPath = args[0]
2771         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2772         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2773         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2774         depotDir = re.sub(r"/$", "", depotDir)
2775         return os.path.split(depotDir)[1]
2776
2777     def run(self, args):
2778         if len(args) < 1:
2779             return False
2780
2781         if self.keepRepoPath and not self.cloneDestination:
2782             sys.stderr.write("Must specify destination for --keep-path\n")
2783             sys.exit(1)
2784
2785         depotPaths = args
2786
2787         if not self.cloneDestination and len(depotPaths) > 1:
2788             self.cloneDestination = depotPaths[-1]
2789             depotPaths = depotPaths[:-1]
2790
2791         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2792         for p in depotPaths:
2793             if not p.startswith("//"):
2794                 return False
2795
2796         if not self.cloneDestination:
2797             self.cloneDestination = self.defaultDestination(args)
2798
2799         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2800
2801         if not os.path.exists(self.cloneDestination):
2802             os.makedirs(self.cloneDestination)
2803         chdir(self.cloneDestination)
2804
2805         init_cmd = [ "git", "init" ]
2806         if self.cloneBare:
2807             init_cmd.append("--bare")
2808         subprocess.check_call(init_cmd)
2809
2810         if not P4Sync.run(self, depotPaths):
2811             return False
2812         if self.branch != "master":
2813             if self.importIntoRemotes:
2814                 masterbranch = "refs/remotes/p4/master"
2815             else:
2816                 masterbranch = "refs/heads/p4/master"
2817             if gitBranchExists(masterbranch):
2818                 system("git branch master %s" % masterbranch)
2819                 if not self.cloneBare:
2820                     system("git checkout -f")
2821             else:
2822                 print "Could not detect main branch. No checkout/master branch created."
2823
2824         # auto-set this variable if invoked with --use-client-spec
2825         if self.useClientSpec_from_options:
2826             system("git config --bool git-p4.useclientspec true")
2827
2828         return True
2829
2830 class P4Branches(Command):
2831     def __init__(self):
2832         Command.__init__(self)
2833         self.options = [ ]
2834         self.description = ("Shows the git branches that hold imports and their "
2835                             + "corresponding perforce depot paths")
2836         self.verbose = False
2837
2838     def run(self, args):
2839         if originP4BranchesExist():
2840             createOrUpdateBranchesFromOrigin()
2841
2842         cmdline = "git rev-parse --symbolic "
2843         cmdline += " --remotes"
2844
2845         for line in read_pipe_lines(cmdline):
2846             line = line.strip()
2847
2848             if not line.startswith('p4/') or line == "p4/HEAD":
2849                 continue
2850             branch = line
2851
2852             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2853             settings = extractSettingsGitLog(log)
2854
2855             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2856         return True
2857
2858 class HelpFormatter(optparse.IndentedHelpFormatter):
2859     def __init__(self):
2860         optparse.IndentedHelpFormatter.__init__(self)
2861
2862     def format_description(self, description):
2863         if description:
2864             return description + "\n"
2865         else:
2866             return ""
2867
2868 def printUsage(commands):
2869     print "usage: %s <command> [options]" % sys.argv[0]
2870     print ""
2871     print "valid commands: %s" % ", ".join(commands)
2872     print ""
2873     print "Try %s <command> --help for command specific help." % sys.argv[0]
2874     print ""
2875
2876 commands = {
2877     "debug" : P4Debug,
2878     "submit" : P4Submit,
2879     "commit" : P4Submit,
2880     "sync" : P4Sync,
2881     "rebase" : P4Rebase,
2882     "clone" : P4Clone,
2883     "rollback" : P4RollBack,
2884     "branches" : P4Branches
2885 }
2886
2887
2888 def main():
2889     if len(sys.argv[1:]) == 0:
2890         printUsage(commands.keys())
2891         sys.exit(2)
2892
2893     cmd = ""
2894     cmdName = sys.argv[1]
2895     try:
2896         klass = commands[cmdName]
2897         cmd = klass()
2898     except KeyError:
2899         print "unknown command %s" % cmdName
2900         print ""
2901         printUsage(commands.keys())
2902         sys.exit(2)
2903
2904     options = cmd.options
2905     cmd.gitdir = os.environ.get("GIT_DIR", None)
2906
2907     args = sys.argv[2:]
2908
2909     options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))
2910     if cmd.needsGit:
2911         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2912
2913     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2914                                    options,
2915                                    description = cmd.description,
2916                                    formatter = HelpFormatter())
2917
2918     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2919     global verbose
2920     verbose = cmd.verbose
2921     if cmd.needsGit:
2922         if cmd.gitdir == None:
2923             cmd.gitdir = os.path.abspath(".git")
2924             if not isValidGitDir(cmd.gitdir):
2925                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2926                 if os.path.exists(cmd.gitdir):
2927                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2928                     if len(cdup) > 0:
2929                         chdir(cdup);
2930
2931         if not isValidGitDir(cmd.gitdir):
2932             if isValidGitDir(cmd.gitdir + "/.git"):
2933                 cmd.gitdir += "/.git"
2934             else:
2935                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2936
2937         os.environ["GIT_DIR"] = cmd.gitdir
2938
2939     if not cmd.run(args):
2940         parser.print_help()
2941         sys.exit(2)
2942
2943
2944 if __name__ == '__main__':
2945     main()