Pileus Git - ~andy/git/blob - git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re, shutil
  14
  15 verbose = False
  16
  17 # Only labels/tags matching this will be imported/exported
  18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = ["p4"]
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += ["-u",user]
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += ["-P", password]
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += ["-p", port]
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += ["-H", host]
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += ["-c", client]
  48
  49
  50     if isinstance(cmd,basestring):
  51         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  52     else:
  53         real_cmd += cmd
  54     return real_cmd
  55
  56 def chdir(dir):
  57     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  58     # not using the shell, we have to set it ourselves.  This path could
  59     # be relative, so go there first, then figure out where we ended up.
  60     os.chdir(dir)
  61     os.environ['PWD'] = os.getcwd()
  62
  63 def die(msg):
  64     if verbose:
  65         raise Exception(msg)
  66     else:
  67         sys.stderr.write(msg + "\n")
  68         sys.exit(1)
  69
  70 def write_pipe(c, stdin):
  71     if verbose:
  72         sys.stderr.write('Writing pipe: %s\n' % str(c))
  73
  74     expand = isinstance(c,basestring)
  75     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  76     pipe = p.stdin
  77     val = pipe.write(stdin)
  78     pipe.close()
  79     if p.wait():
  80         die('Command failed: %s' % str(c))
  81
  82     return val
  83
  84 def p4_write_pipe(c, stdin):
  85     real_cmd = p4_build_cmd(c)
  86     return write_pipe(real_cmd, stdin)
  87
  88 def read_pipe(c, ignore_error=False):
  89     if verbose:
  90         sys.stderr.write('Reading pipe: %s\n' % str(c))
  91
  92     expand = isinstance(c,basestring)
  93     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  94     pipe = p.stdout
  95     val = pipe.read()
  96     if p.wait() and not ignore_error:
  97         die('Command failed: %s' % str(c))
  98
  99     return val
 100
 101 def p4_read_pipe(c, ignore_error=False):
 102     real_cmd = p4_build_cmd(c)
 103     return read_pipe(real_cmd, ignore_error)
 104
 105 def read_pipe_lines(c):
 106     if verbose:
 107         sys.stderr.write('Reading pipe: %s\n' % str(c))
 108
 109     expand = isinstance(c, basestring)
 110     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 111     pipe = p.stdout
 112     val = pipe.readlines()
 113     if pipe.close() or p.wait():
 114         die('Command failed: %s' % str(c))
 115
 116     return val
 117
 118 def p4_read_pipe_lines(c):
 119     """Specifically invoke p4 on the command supplied. """
 120     real_cmd = p4_build_cmd(c)
 121     return read_pipe_lines(real_cmd)
 122
 123 def system(cmd):
 124     expand = isinstance(cmd,basestring)
 125     if verbose:
 126         sys.stderr.write("executing %s\n" % str(cmd))
 127     subprocess.check_call(cmd, shell=expand)
 128
 129 def p4_system(cmd):
 130     """Specifically invoke p4 as the system command. """
 131     real_cmd = p4_build_cmd(cmd)
 132     expand = isinstance(real_cmd, basestring)
 133     subprocess.check_call(real_cmd, shell=expand)
 134
 135 def p4_integrate(src, dest):
 136     p4_system(["integrate", "-Dt", src, dest])
 137
 138 def p4_sync(path):
 139     p4_system(["sync", path])
 140
 141 def p4_add(f):
 142     p4_system(["add", f])
 143
 144 def p4_delete(f):
 145     p4_system(["delete", f])
 146
 147 def p4_edit(f):
 148     p4_system(["edit", f])
 149
 150 def p4_revert(f):
 151     p4_system(["revert", f])
 152
 153 def p4_reopen(type, file):
 154     p4_system(["reopen", "-t", type, file])
 155
 156 #
 157 # Canonicalize the p4 type and return a tuple of the
 158 # base type, plus any modifiers.  See "p4 help filetypes"
 159 # for a list and explanation.
 160 #
 161 def split_p4_type(p4type):
 162
 163     p4_filetypes_historical = {
 164         "ctempobj": "binary+Sw",
 165         "ctext": "text+C",
 166         "cxtext": "text+Cx",
 167         "ktext": "text+k",
 168         "kxtext": "text+kx",
 169         "ltext": "text+F",
 170         "tempobj": "binary+FSw",
 171         "ubinary": "binary+F",
 172         "uresource": "resource+F",
 173         "uxbinary": "binary+Fx",
 174         "xbinary": "binary+x",
 175         "xltext": "text+Fx",
 176         "xtempobj": "binary+Swx",
 177         "xtext": "text+x",
 178         "xunicode": "unicode+x",
 179         "xutf16": "utf16+x",
 180     }
 181     if p4type in p4_filetypes_historical:
 182         p4type = p4_filetypes_historical[p4type]
 183     mods = ""
 184     s = p4type.split("+")
 185     base = s[0]
 186     mods = ""
 187     if len(s) > 1:
 188         mods = s[1]
 189     return (base, mods)
 190
 191 #
 192 # return the raw p4 type of a file (text, text+ko, etc)
 193 #
 194 def p4_type(file):
 195     results = p4CmdList(["fstat", "-T", "headType", file])
 196     return results[0]['headType']
 197
 198 #
 199 # Given a type base and modifier, return a regexp matching
 200 # the keywords that can be expanded in the file
 201 #
 202 def p4_keywords_regexp_for_type(base, type_mods):
 203     if base in ("text", "unicode", "binary"):
 204         kwords = None
 205         if "ko" in type_mods:
 206             kwords = 'Id|Header'
 207         elif "k" in type_mods:
 208             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 209         else:
 210             return None
 211         pattern = r"""
 212             \$              # Starts with a dollar, followed by...
 213             (%s)            # one of the keywords, followed by...
 214             (:[^$]+)?       # possibly an old expansion, followed by...
 215             \$              # another dollar
 216             """ % kwords
 217         return pattern
 218     else:
 219         return None
 220
 221 #
 222 # Given a file, return a regexp matching the possible
 223 # RCS keywords that will be expanded, or None for files
 224 # with kw expansion turned off.
 225 #
 226 def p4_keywords_regexp_for_file(file):
 227     if not os.path.exists(file):
 228         return None
 229     else:
 230         (type_base, type_mods) = split_p4_type(p4_type(file))
 231         return p4_keywords_regexp_for_type(type_base, type_mods)
 232
 233 def setP4ExecBit(file, mode):
 234     # Reopens an already open file and changes the execute bit to match
 235     # the execute bit setting in the passed in mode.
 236
 237     p4Type = "+x"
 238
 239     if not isModeExec(mode):
 240         p4Type = getP4OpenedType(file)
 241         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 242         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 243         if p4Type[-1] == "+":
 244             p4Type = p4Type[0:-1]
 245
 246     p4_reopen(p4Type, file)
 247
 248 def getP4OpenedType(file):
 249     # Returns the perforce file type for the given file.
 250
 251     result = p4_read_pipe(["opened", file])
 252     match = re.match(".*\((.+)\)\r?$", result)
 253     if match:
 254         return match.group(1)
 255     else:
 256         die("Could not determine file type for %s (result: '%s')" % (file, result))
 257
 258 # Return the set of all p4 labels
 259 def getP4Labels(depotPaths):
 260     labels = set()
 261     if isinstance(depotPaths,basestring):
 262         depotPaths = [depotPaths]
 263
 264     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 265         label = l['label']
 266         labels.add(label)
 267
 268     return labels
 269
 270 # Return the set of all git tags
 271 def getGitTags():
 272     gitTags = set()
 273     for line in read_pipe_lines(["git", "tag"]):
 274         tag = line.strip()
 275         gitTags.add(tag)
 276     return gitTags
 277
 278 def diffTreePattern():
 279     # This is a simple generator for the diff tree regex pattern. This could be
 280     # a class variable if this and parseDiffTreeEntry were a part of a class.
 281     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 282     while True:
 283         yield pattern
 284
 285 def parseDiffTreeEntry(entry):
 286     """Parses a single diff tree entry into its component elements.
 287
 288     See git-diff-tree(1) manpage for details about the format of the diff
 289     output. This method returns a dictionary with the following elements:
 290
 291     src_mode - The mode of the source file
 292     dst_mode - The mode of the destination file
 293     src_sha1 - The sha1 for the source file
 294     dst_sha1 - The sha1 fr the destination file
 295     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 296     status_score - The score for the status (applicable for 'C' and 'R'
 297                    statuses). This is None if there is no score.
 298     src - The path for the source file.
 299     dst - The path for the destination file. This is only present for
 300           copy or renames. If it is not present, this is None.
 301
 302     If the pattern is not matched, None is returned."""
 303
 304     match = diffTreePattern().next().match(entry)
 305     if match:
 306         return {
 307             'src_mode': match.group(1),
 308             'dst_mode': match.group(2),
 309             'src_sha1': match.group(3),
 310             'dst_sha1': match.group(4),
 311             'status': match.group(5),
 312             'status_score': match.group(6),
 313             'src': match.group(7),
 314             'dst': match.group(10)
 315         }
 316     return None
 317
 318 def isModeExec(mode):
 319     # Returns True if the given git mode represents an executable file,
 320     # otherwise False.
 321     return mode[-3:] == "755"
 322
 323 def isModeExecChanged(src_mode, dst_mode):
 324     return isModeExec(src_mode) != isModeExec(dst_mode)
 325
 326 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 327
 328     if isinstance(cmd,basestring):
 329         cmd = "-G " + cmd
 330         expand = True
 331     else:
 332         cmd = ["-G"] + cmd
 333         expand = False
 334
 335     cmd = p4_build_cmd(cmd)
 336     if verbose:
 337         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 338
 339     # Use a temporary file to avoid deadlocks without
 340     # subprocess.communicate(), which would put another copy
 341     # of stdout into memory.
 342     stdin_file = None
 343     if stdin is not None:
 344         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 345         if isinstance(stdin,basestring):
 346             stdin_file.write(stdin)
 347         else:
 348             for i in stdin:
 349                 stdin_file.write(i + '\n')
 350         stdin_file.flush()
 351         stdin_file.seek(0)
 352
 353     p4 = subprocess.Popen(cmd,
 354                           shell=expand,
 355                           stdin=stdin_file,
 356                           stdout=subprocess.PIPE)
 357
 358     result = []
 359     try:
 360         while True:
 361             entry = marshal.load(p4.stdout)
 362             if cb is not None:
 363                 cb(entry)
 364             else:
 365                 result.append(entry)
 366     except EOFError:
 367         pass
 368     exitCode = p4.wait()
 369     if exitCode != 0:
 370         entry = {}
 371         entry["p4ExitCode"] = exitCode
 372         result.append(entry)
 373
 374     return result
 375
 376 def p4Cmd(cmd):
 377     list = p4CmdList(cmd)
 378     result = {}
 379     for entry in list:
 380         result.update(entry)
 381     return result;
 382
 383 def p4Where(depotPath):
 384     if not depotPath.endswith("/"):
 385         depotPath += "/"
 386     depotPath = depotPath + "..."
 387     outputList = p4CmdList(["where", depotPath])
 388     output = None
 389     for entry in outputList:
 390         if "depotFile" in entry:
 391             if entry["depotFile"] == depotPath:
 392                 output = entry
 393                 break
 394         elif "data" in entry:
 395             data = entry.get("data")
 396             space = data.find(" ")
 397             if data[:space] == depotPath:
 398                 output = entry
 399                 break
 400     if output == None:
 401         return ""
 402     if output["code"] == "error":
 403         return ""
 404     clientPath = ""
 405     if "path" in output:
 406         clientPath = output.get("path")
 407     elif "data" in output:
 408         data = output.get("data")
 409         lastSpace = data.rfind(" ")
 410         clientPath = data[lastSpace + 1:]
 411
 412     if clientPath.endswith("..."):
 413         clientPath = clientPath[:-3]
 414     return clientPath
 415
 416 def currentGitBranch():
 417     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 418
 419 def isValidGitDir(path):
 420     if (os.path.exists(path + "/HEAD")
 421         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 422         return True;
 423     return False
 424
 425 def parseRevision(ref):
 426     return read_pipe("git rev-parse %s" % ref).strip()
 427
 428 def branchExists(ref):
 429     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 430                      ignore_error=True)
 431     return len(rev) > 0
 432
 433 def extractLogMessageFromGitCommit(commit):
 434     logMessage = ""
 435
 436     ## fixme: title is first line of commit, not 1st paragraph.
 437     foundTitle = False
 438     for log in read_pipe_lines("git cat-file commit %s" % commit):
 439        if not foundTitle:
 440            if len(log) == 1:
 441                foundTitle = True
 442            continue
 443
 444        logMessage += log
 445     return logMessage
 446
 447 def extractSettingsGitLog(log):
 448     values = {}
 449     for line in log.split("\n"):
 450         line = line.strip()
 451         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 452         if not m:
 453             continue
 454
 455         assignments = m.group(1).split (':')
 456         for a in assignments:
 457             vals = a.split ('=')
 458             key = vals[0].strip()
 459             val = ('='.join (vals[1:])).strip()
 460             if val.endswith ('\"') and val.startswith('"'):
 461                 val = val[1:-1]
 462
 463             values[key] = val
 464
 465     paths = values.get("depot-paths")
 466     if not paths:
 467         paths = values.get("depot-path")
 468     if paths:
 469         values['depot-paths'] = paths.split(',')
 470     return values
 471
 472 def gitBranchExists(branch):
 473     proc = subprocess.Popen(["git", "rev-parse", branch],
 474                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 475     return proc.wait() == 0;
 476
 477 _gitConfig = {}
 478 def gitConfig(key, args = None): # set args to "--bool", for instance
 479     if not _gitConfig.has_key(key):
 480         argsFilter = ""
 481         if args != None:
 482             argsFilter = "%s " % args
 483         cmd = "git config %s%s" % (argsFilter, key)
 484         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 485     return _gitConfig[key]
 486
 487 def gitConfigList(key):
 488     if not _gitConfig.has_key(key):
 489         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 490     return _gitConfig[key]
 491
 492 def p4BranchesInGit(branchesAreInRemotes = True):
 493     branches = {}
 494
 495     cmdline = "git rev-parse --symbolic "
 496     if branchesAreInRemotes:
 497         cmdline += " --remotes"
 498     else:
 499         cmdline += " --branches"
 500
 501     for line in read_pipe_lines(cmdline):
 502         line = line.strip()
 503
 504         ## only import to p4/
 505         if not line.startswith('p4/') or line == "p4/HEAD":
 506             continue
 507         branch = line
 508
 509         # strip off p4
 510         branch = re.sub ("^p4/", "", line)
 511
 512         branches[branch] = parseRevision(line)
 513     return branches
 514
 515 def findUpstreamBranchPoint(head = "HEAD"):
 516     branches = p4BranchesInGit()
 517     # map from depot-path to branch name
 518     branchByDepotPath = {}
 519     for branch in branches.keys():
 520         tip = branches[branch]
 521         log = extractLogMessageFromGitCommit(tip)
 522         settings = extractSettingsGitLog(log)
 523         if settings.has_key("depot-paths"):
 524             paths = ",".join(settings["depot-paths"])
 525             branchByDepotPath[paths] = "remotes/p4/" + branch
 526
 527     settings = None
 528     parent = 0
 529     while parent < 65535:
 530         commit = head + "~%s" % parent
 531         log = extractLogMessageFromGitCommit(commit)
 532         settings = extractSettingsGitLog(log)
 533         if settings.has_key("depot-paths"):
 534             paths = ",".join(settings["depot-paths"])
 535             if branchByDepotPath.has_key(paths):
 536                 return [branchByDepotPath[paths], settings]
 537
 538         parent = parent + 1
 539
 540     return ["", settings]
 541
 542 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 543     if not silent:
 544         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 545                % localRefPrefix)
 546
 547     originPrefix = "origin/p4/"
 548
 549     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 550         line = line.strip()
 551         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 552             continue
 553
 554         headName = line[len(originPrefix):]
 555         remoteHead = localRefPrefix + headName
 556         originHead = line
 557
 558         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 559         if (not original.has_key('depot-paths')
 560             or not original.has_key('change')):
 561             continue
 562
 563         update = False
 564         if not gitBranchExists(remoteHead):
 565             if verbose:
 566                 print "creating %s" % remoteHead
 567             update = True
 568         else:
 569             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 570             if settings.has_key('change') > 0:
 571                 if settings['depot-paths'] == original['depot-paths']:
 572                     originP4Change = int(original['change'])
 573                     p4Change = int(settings['change'])
 574                     if originP4Change > p4Change:
 575                         print ("%s (%s) is newer than %s (%s). "
 576                                "Updating p4 branch from origin."
 577                                % (originHead, originP4Change,
 578                                   remoteHead, p4Change))
 579                         update = True
 580                 else:
 581                     print ("Ignoring: %s was imported from %s while "
 582                            "%s was imported from %s"
 583                            % (originHead, ','.join(original['depot-paths']),
 584                               remoteHead, ','.join(settings['depot-paths'])))
 585
 586         if update:
 587             system("git update-ref %s %s" % (remoteHead, originHead))
 588
 589 def originP4BranchesExist():
 590         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 591
 592 def p4ChangesForPaths(depotPaths, changeRange):
 593     assert depotPaths
 594     cmd = ['changes']
 595     for p in depotPaths:
 596         cmd += ["%s...%s" % (p, changeRange)]
 597     output = p4_read_pipe_lines(cmd)
 598
 599     changes = {}
 600     for line in output:
 601         changeNum = int(line.split(" ")[1])
 602         changes[changeNum] = True
 603
 604     changelist = changes.keys()
 605     changelist.sort()
 606     return changelist
 607
 608 def p4PathStartsWith(path, prefix):
 609     # This method tries to remedy a potential mixed-case issue:
 610     #
 611     # If UserA adds  //depot/DirA/file1
 612     # and UserB adds //depot/dira/file2
 613     #
 614     # we may or may not have a problem. If you have core.ignorecase=true,
 615     # we treat DirA and dira as the same directory
 616     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 617     if ignorecase:
 618         return path.lower().startswith(prefix.lower())
 619     return path.startswith(prefix)
 620
 621 def getClientSpec():
 622     """Look at the p4 client spec, create a View() object that contains
 623        all the mappings, and return it."""
 624
 625     specList = p4CmdList("client -o")
 626     if len(specList) != 1:
 627         die('Output from "client -o" is %d lines, expecting 1' %
 628             len(specList))
 629
 630     # dictionary of all client parameters
 631     entry = specList[0]
 632
 633     # just the keys that start with "View"
 634     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 635
 636     # hold this new View
 637     view = View()
 638
 639     # append the lines, in order, to the view
 640     for view_num in range(len(view_keys)):
 641         k = "View%d" % view_num
 642         if k not in view_keys:
 643             die("Expected view key %s missing" % k)
 644         view.append(entry[k])
 645
 646     return view
 647
 648 def getClientRoot():
 649     """Grab the client directory."""
 650
 651     output = p4CmdList("client -o")
 652     if len(output) != 1:
 653         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 654
 655     entry = output[0]
 656     if "Root" not in entry:
 657         die('Client has no "Root"')
 658
 659     return entry["Root"]
 660
 661 class Command:
 662     def __init__(self):
 663         self.usage = "usage: %prog [options]"
 664         self.needsGit = True
 665         self.verbose = False
 666
 667 class P4UserMap:
 668     def __init__(self):
 669         self.userMapFromPerforceServer = False
 670         self.myP4UserId = None
 671
 672     def p4UserId(self):
 673         if self.myP4UserId:
 674             return self.myP4UserId
 675
 676         results = p4CmdList("user -o")
 677         for r in results:
 678             if r.has_key('User'):
 679                 self.myP4UserId = r['User']
 680                 return r['User']
 681         die("Could not find your p4 user id")
 682
 683     def p4UserIsMe(self, p4User):
 684         # return True if the given p4 user is actually me
 685         me = self.p4UserId()
 686         if not p4User or p4User != me:
 687             return False
 688         else:
 689             return True
 690
 691     def getUserCacheFilename(self):
 692         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 693         return home + "/.gitp4-usercache.txt"
 694
 695     def getUserMapFromPerforceServer(self):
 696         if self.userMapFromPerforceServer:
 697             return
 698         self.users = {}
 699         self.emails = {}
 700
 701         for output in p4CmdList("users"):
 702             if not output.has_key("User"):
 703                 continue
 704             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 705             self.emails[output["Email"]] = output["User"]
 706
 707
 708         s = ''
 709         for (key, val) in self.users.items():
 710             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 711
 712         open(self.getUserCacheFilename(), "wb").write(s)
 713         self.userMapFromPerforceServer = True
 714
 715     def loadUserMapFromCache(self):
 716         self.users = {}
 717         self.userMapFromPerforceServer = False
 718         try:
 719             cache = open(self.getUserCacheFilename(), "rb")
 720             lines = cache.readlines()
 721             cache.close()
 722             for line in lines:
 723                 entry = line.strip().split("\t")
 724                 self.users[entry[0]] = entry[1]
 725         except IOError:
 726             self.getUserMapFromPerforceServer()
 727
 728 class P4Debug(Command):
 729     def __init__(self):
 730         Command.__init__(self)
 731         self.options = []
 732         self.description = "A tool to debug the output of p4 -G."
 733         self.needsGit = False
 734
 735     def run(self, args):
 736         j = 0
 737         for output in p4CmdList(args):
 738             print 'Element: %d' % j
 739             j += 1
 740             print output
 741         return True
 742
 743 class P4RollBack(Command):
 744     def __init__(self):
 745         Command.__init__(self)
 746         self.options = [
 747             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 748         ]
 749         self.description = "A tool to debug the multi-branch import. Don't use :)"
 750         self.rollbackLocalBranches = False
 751
 752     def run(self, args):
 753         if len(args) != 1:
 754             return False
 755         maxChange = int(args[0])
 756
 757         if "p4ExitCode" in p4Cmd("changes -m 1"):
 758             die("Problems executing p4");
 759
 760         if self.rollbackLocalBranches:
 761             refPrefix = "refs/heads/"
 762             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 763         else:
 764             refPrefix = "refs/remotes/"
 765             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 766
 767         for line in lines:
 768             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 769                 line = line.strip()
 770                 ref = refPrefix + line
 771                 log = extractLogMessageFromGitCommit(ref)
 772                 settings = extractSettingsGitLog(log)
 773
 774                 depotPaths = settings['depot-paths']
 775                 change = settings['change']
 776
 777                 changed = False
 778
 779                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 780                                                            for p in depotPaths]))) == 0:
 781                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 782                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 783                     continue
 784
 785                 while change and int(change) > maxChange:
 786                     changed = True
 787                     if self.verbose:
 788                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 789                     system("git update-ref %s \"%s^\"" % (ref, ref))
 790                     log = extractLogMessageFromGitCommit(ref)
 791                     settings =  extractSettingsGitLog(log)
 792
 793
 794                     depotPaths = settings['depot-paths']
 795                     change = settings['change']
 796
 797                 if changed:
 798                     print "%s rewound to %s" % (ref, change)
 799
 800         return True
 801
 802 class P4Submit(Command, P4UserMap):
 803     def __init__(self):
 804         Command.__init__(self)
 805         P4UserMap.__init__(self)
 806         self.options = [
 807                 optparse.make_option("--origin", dest="origin"),
 808                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 809                 # preserve the user, requires relevant p4 permissions
 810                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 811                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 812         ]
 813         self.description = "Submit changes from git to the perforce depot."
 814         self.usage += " [name of git branch to submit into perforce depot]"
 815         self.interactive = True
 816         self.origin = ""
 817         self.detectRenames = False
 818         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 819         self.isWindows = (platform.system() == "Windows")
 820         self.exportLabels = False
 821
 822     def check(self):
 823         if len(p4CmdList("opened ...")) > 0:
 824             die("You have files opened with perforce! Close them before starting the sync.")
 825
 826     # replaces everything between 'Description:' and the next P4 submit template field with the
 827     # commit message
 828     def prepareLogMessage(self, template, message):
 829         result = ""
 830
 831         inDescriptionSection = False
 832
 833         for line in template.split("\n"):
 834             if line.startswith("#"):
 835                 result += line + "\n"
 836                 continue
 837
 838             if inDescriptionSection:
 839                 if line.startswith("Files:") or line.startswith("Jobs:"):
 840                     inDescriptionSection = False
 841                 else:
 842                     continue
 843             else:
 844                 if line.startswith("Description:"):
 845                     inDescriptionSection = True
 846                     line += "\n"
 847                     for messageLine in message.split("\n"):
 848                         line += "\t" + messageLine + "\n"
 849
 850             result += line + "\n"
 851
 852         return result
 853
 854     def patchRCSKeywords(self, file, pattern):
 855         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
 856         (handle, outFileName) = tempfile.mkstemp(dir='.')
 857         try:
 858             outFile = os.fdopen(handle, "w+")
 859             inFile = open(file, "r")
 860             regexp = re.compile(pattern, re.VERBOSE)
 861             for line in inFile.readlines():
 862                 line = regexp.sub(r'$\1$', line)
 863                 outFile.write(line)
 864             inFile.close()
 865             outFile.close()
 866             # Forcibly overwrite the original file
 867             os.unlink(file)
 868             shutil.move(outFileName, file)
 869         except:
 870             # cleanup our temporary file
 871             os.unlink(outFileName)
 872             print "Failed to strip RCS keywords in %s" % file
 873             raise
 874
 875         print "Patched up RCS keywords in %s" % file
 876
 877     def p4UserForCommit(self,id):
 878         # Return the tuple (perforce user,git email) for a given git commit id
 879         self.getUserMapFromPerforceServer()
 880         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 881         gitEmail = gitEmail.strip()
 882         if not self.emails.has_key(gitEmail):
 883             return (None,gitEmail)
 884         else:
 885             return (self.emails[gitEmail],gitEmail)
 886
 887     def checkValidP4Users(self,commits):
 888         # check if any git authors cannot be mapped to p4 users
 889         for id in commits:
 890             (user,email) = self.p4UserForCommit(id)
 891             if not user:
 892                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 893                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 894                     print "%s" % msg
 895                 else:
 896                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 897
 898     def lastP4Changelist(self):
 899         # Get back the last changelist number submitted in this client spec. This
 900         # then gets used to patch up the username in the change. If the same
 901         # client spec is being used by multiple processes then this might go
 902         # wrong.
 903         results = p4CmdList("client -o")        # find the current client
 904         client = None
 905         for r in results:
 906             if r.has_key('Client'):
 907                 client = r['Client']
 908                 break
 909         if not client:
 910             die("could not get client spec")
 911         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 912         for r in results:
 913             if r.has_key('change'):
 914                 return r['change']
 915         die("Could not get changelist number for last submit - cannot patch up user details")
 916
 917     def modifyChangelistUser(self, changelist, newUser):
 918         # fixup the user field of a changelist after it has been submitted.
 919         changes = p4CmdList("change -o %s" % changelist)
 920         if len(changes) != 1:
 921             die("Bad output from p4 change modifying %s to user %s" %
 922                 (changelist, newUser))
 923
 924         c = changes[0]
 925         if c['User'] == newUser: return   # nothing to do
 926         c['User'] = newUser
 927         input = marshal.dumps(c)
 928
 929         result = p4CmdList("change -f -i", stdin=input)
 930         for r in result:
 931             if r.has_key('code'):
 932                 if r['code'] == 'error':
 933                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 934             if r.has_key('data'):
 935                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 936                 return
 937         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 938
 939     def canChangeChangelists(self):
 940         # check to see if we have p4 admin or super-user permissions, either of
 941         # which are required to modify changelists.
 942         results = p4CmdList(["protects", self.depotPath])
 943         for r in results:
 944             if r.has_key('perm'):
 945                 if r['perm'] == 'admin':
 946                     return 1
 947                 if r['perm'] == 'super':
 948                     return 1
 949         return 0
 950
 951     def prepareSubmitTemplate(self):
 952         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 953         template = ""
 954         inFilesSection = False
 955         for line in p4_read_pipe_lines(['change', '-o']):
 956             if line.endswith("\r\n"):
 957                 line = line[:-2] + "\n"
 958             if inFilesSection:
 959                 if line.startswith("\t"):
 960                     # path starts and ends with a tab
 961                     path = line[1:]
 962                     lastTab = path.rfind("\t")
 963                     if lastTab != -1:
 964                         path = path[:lastTab]
 965                         if not p4PathStartsWith(path, self.depotPath):
 966                             continue
 967                 else:
 968                     inFilesSection = False
 969             else:
 970                 if line.startswith("Files:"):
 971                     inFilesSection = True
 972
 973             template += line
 974
 975         return template
 976
 977     def edit_template(self, template_file):
 978         """Invoke the editor to let the user change the submission
 979            message.  Return true if okay to continue with the submit."""
 980
 981         # if configured to skip the editing part, just submit
 982         if gitConfig("git-p4.skipSubmitEdit") == "true":
 983             return True
 984
 985         # look at the modification time, to check later if the user saved
 986         # the file
 987         mtime = os.stat(template_file).st_mtime
 988
 989         # invoke the editor
 990         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
 991             editor = os.environ.get("P4EDITOR")
 992         else:
 993             editor = read_pipe("git var GIT_EDITOR").strip()
 994         system(editor + " " + template_file)
 995
 996         # If the file was not saved, prompt to see if this patch should
 997         # be skipped.  But skip this verification step if configured so.
 998         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 999             return True
1000
1001         # modification time updated means user saved the file
1002         if os.stat(template_file).st_mtime > mtime:
1003             return True
1004
1005         while True:
1006             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1007             if response == 'y':
1008                 return True
1009             if response == 'n':
1010                 return False
1011
1012     def applyCommit(self, id):
1013         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1014
1015         (p4User, gitEmail) = self.p4UserForCommit(id)
1016
1017         if not self.detectRenames:
1018             # If not explicitly set check the config variable
1019             self.detectRenames = gitConfig("git-p4.detectRenames")
1020
1021         if self.detectRenames.lower() == "false" or self.detectRenames == "":
1022             diffOpts = ""
1023         elif self.detectRenames.lower() == "true":
1024             diffOpts = "-M"
1025         else:
1026             diffOpts = "-M%s" % self.detectRenames
1027
1028         detectCopies = gitConfig("git-p4.detectCopies")
1029         if detectCopies.lower() == "true":
1030             diffOpts += " -C"
1031         elif detectCopies != "" and detectCopies.lower() != "false":
1032             diffOpts += " -C%s" % detectCopies
1033
1034         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1035             diffOpts += " --find-copies-harder"
1036
1037         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
1038         filesToAdd = set()
1039         filesToDelete = set()
1040         editedFiles = set()
1041         filesToChangeExecBit = {}
1042
1043         for line in diff:
1044             diff = parseDiffTreeEntry(line)
1045             modifier = diff['status']
1046             path = diff['src']
1047             if modifier == "M":
1048                 p4_edit(path)
1049                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1050                     filesToChangeExecBit[path] = diff['dst_mode']
1051                 editedFiles.add(path)
1052             elif modifier == "A":
1053                 filesToAdd.add(path)
1054                 filesToChangeExecBit[path] = diff['dst_mode']
1055                 if path in filesToDelete:
1056                     filesToDelete.remove(path)
1057             elif modifier == "D":
1058                 filesToDelete.add(path)
1059                 if path in filesToAdd:
1060                     filesToAdd.remove(path)
1061             elif modifier == "C":
1062                 src, dest = diff['src'], diff['dst']
1063                 p4_integrate(src, dest)
1064                 if diff['src_sha1'] != diff['dst_sha1']:
1065                     p4_edit(dest)
1066                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1067                     p4_edit(dest)
1068                     filesToChangeExecBit[dest] = diff['dst_mode']
1069                 os.unlink(dest)
1070                 editedFiles.add(dest)
1071             elif modifier == "R":
1072                 src, dest = diff['src'], diff['dst']
1073                 p4_integrate(src, dest)
1074                 if diff['src_sha1'] != diff['dst_sha1']:
1075                     p4_edit(dest)
1076                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1077                     p4_edit(dest)
1078                     filesToChangeExecBit[dest] = diff['dst_mode']
1079                 os.unlink(dest)
1080                 editedFiles.add(dest)
1081                 filesToDelete.add(src)
1082             else:
1083                 die("unknown modifier %s for %s" % (modifier, path))
1084
1085         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1086         patchcmd = diffcmd + " | git apply "
1087         tryPatchCmd = patchcmd + "--check -"
1088         applyPatchCmd = patchcmd + "--check --apply -"
1089         patch_succeeded = True
1090
1091         if os.system(tryPatchCmd) != 0:
1092             fixed_rcs_keywords = False
1093             patch_succeeded = False
1094             print "Unfortunately applying the change failed!"
1095
1096             # Patch failed, maybe it's just RCS keyword woes. Look through
1097             # the patch to see if that's possible.
1098             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1099                 file = None
1100                 pattern = None
1101                 kwfiles = {}
1102                 for file in editedFiles | filesToDelete:
1103                     # did this file's delta contain RCS keywords?
1104                     pattern = p4_keywords_regexp_for_file(file)
1105
1106                     if pattern:
1107                         # this file is a possibility...look for RCS keywords.
1108                         regexp = re.compile(pattern, re.VERBOSE)
1109                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1110                             if regexp.search(line):
1111                                 if verbose:
1112                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1113                                 kwfiles[file] = pattern
1114                                 break
1115
1116                 for file in kwfiles:
1117                     if verbose:
1118                         print "zapping %s with %s" % (line,pattern)
1119                     self.patchRCSKeywords(file, kwfiles[file])
1120                     fixed_rcs_keywords = True
1121
1122             if fixed_rcs_keywords:
1123                 print "Retrying the patch with RCS keywords cleaned up"
1124                 if os.system(tryPatchCmd) == 0:
1125                     patch_succeeded = True
1126
1127         if not patch_succeeded:
1128             print "What do you want to do?"
1129             response = "x"
1130             while response != "s" and response != "a" and response != "w":
1131                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1132                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1133             if response == "s":
1134                 print "Skipping! Good luck with the next patches..."
1135                 for f in editedFiles:
1136                     p4_revert(f)
1137                 for f in filesToAdd:
1138                     os.remove(f)
1139                 return
1140             elif response == "a":
1141                 os.system(applyPatchCmd)
1142                 if len(filesToAdd) > 0:
1143                     print "You may also want to call p4 add on the following files:"
1144                     print " ".join(filesToAdd)
1145                 if len(filesToDelete):
1146                     print "The following files should be scheduled for deletion with p4 delete:"
1147                     print " ".join(filesToDelete)
1148                 die("Please resolve and submit the conflict manually and "
1149                     + "continue afterwards with git p4 submit --continue")
1150             elif response == "w":
1151                 system(diffcmd + " > patch.txt")
1152                 print "Patch saved to patch.txt in %s !" % self.clientPath
1153                 die("Please resolve and submit the conflict manually and "
1154                     "continue afterwards with git p4 submit --continue")
1155
1156         system(applyPatchCmd)
1157
1158         for f in filesToAdd:
1159             p4_add(f)
1160         for f in filesToDelete:
1161             p4_revert(f)
1162             p4_delete(f)
1163
1164         # Set/clear executable bits
1165         for f in filesToChangeExecBit.keys():
1166             mode = filesToChangeExecBit[f]
1167             setP4ExecBit(f, mode)
1168
1169         logMessage = extractLogMessageFromGitCommit(id)
1170         logMessage = logMessage.strip()
1171
1172         template = self.prepareSubmitTemplate()
1173
1174         if self.interactive:
1175             submitTemplate = self.prepareLogMessage(template, logMessage)
1176
1177             if self.preserveUser:
1178                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1179
1180             if os.environ.has_key("P4DIFF"):
1181                 del(os.environ["P4DIFF"])
1182             diff = ""
1183             for editedFile in editedFiles:
1184                 diff += p4_read_pipe(['diff', '-du', editedFile])
1185
1186             newdiff = ""
1187             for newFile in filesToAdd:
1188                 newdiff += "==== new file ====\n"
1189                 newdiff += "--- /dev/null\n"
1190                 newdiff += "+++ %s\n" % newFile
1191                 f = open(newFile, "r")
1192                 for line in f.readlines():
1193                     newdiff += "+" + line
1194                 f.close()
1195
1196             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1197                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1198                 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1199                 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1200
1201             separatorLine = "######## everything below this line is just the diff #######\n"
1202
1203             (handle, fileName) = tempfile.mkstemp()
1204             tmpFile = os.fdopen(handle, "w+")
1205             if self.isWindows:
1206                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1207                 separatorLine = separatorLine.replace("\n", "\r\n")
1208                 newdiff = newdiff.replace("\n", "\r\n")
1209             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1210             tmpFile.close()
1211
1212             if self.edit_template(fileName):
1213                 # read the edited message and submit
1214                 tmpFile = open(fileName, "rb")
1215                 message = tmpFile.read()
1216                 tmpFile.close()
1217                 submitTemplate = message[:message.index(separatorLine)]
1218                 if self.isWindows:
1219                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1220                 p4_write_pipe(['submit', '-i'], submitTemplate)
1221
1222                 if self.preserveUser:
1223                     if p4User:
1224                         # Get last changelist number. Cannot easily get it from
1225                         # the submit command output as the output is
1226                         # unmarshalled.
1227                         changelist = self.lastP4Changelist()
1228                         self.modifyChangelistUser(changelist, p4User)
1229             else:
1230                 # skip this patch
1231                 print "Submission cancelled, undoing p4 changes."
1232                 for f in editedFiles:
1233                     p4_revert(f)
1234                 for f in filesToAdd:
1235                     p4_revert(f)
1236                     os.remove(f)
1237
1238             os.remove(fileName)
1239         else:
1240             fileName = "submit.txt"
1241             file = open(fileName, "w+")
1242             file.write(self.prepareLogMessage(template, logMessage))
1243             file.close()
1244             print ("Perforce submit template written as %s. "
1245                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1246                    % (fileName, fileName))
1247
1248     # Export git tags as p4 labels. Create a p4 label and then tag
1249     # with that.
1250     def exportGitTags(self, gitTags):
1251         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1252         if len(validLabelRegexp) == 0:
1253             validLabelRegexp = defaultLabelRegexp
1254         m = re.compile(validLabelRegexp)
1255
1256         for name in gitTags:
1257
1258             if not m.match(name):
1259                 if verbose:
1260                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1261                 continue
1262
1263             # Get the p4 commit this corresponds to
1264             logMessage = extractLogMessageFromGitCommit(name)
1265             values = extractSettingsGitLog(logMessage)
1266
1267             if not values.has_key('change'):
1268                 # a tag pointing to something not sent to p4; ignore
1269                 if verbose:
1270                     print "git tag %s does not give a p4 commit" % name
1271                 continue
1272             else:
1273                 changelist = values['change']
1274
1275             # Get the tag details.
1276             inHeader = True
1277             isAnnotated = False
1278             body = []
1279             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1280                 l = l.strip()
1281                 if inHeader:
1282                     if re.match(r'tag\s+', l):
1283                         isAnnotated = True
1284                     elif re.match(r'\s*$', l):
1285                         inHeader = False
1286                         continue
1287                 else:
1288                     body.append(l)
1289
1290             if not isAnnotated:
1291                 body = ["lightweight tag imported by git p4\n"]
1292
1293             # Create the label - use the same view as the client spec we are using
1294             clientSpec = getClientSpec()
1295
1296             labelTemplate  = "Label: %s\n" % name
1297             labelTemplate += "Description:\n"
1298             for b in body:
1299                 labelTemplate += "\t" + b + "\n"
1300             labelTemplate += "View:\n"
1301             for mapping in clientSpec.mappings:
1302                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1303
1304             p4_write_pipe(["label", "-i"], labelTemplate)
1305
1306             # Use the label
1307             p4_system(["tag", "-l", name] +
1308                       ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1309
1310             if verbose:
1311                 print "created p4 label for tag %s" % name
1312
1313     def run(self, args):
1314         if len(args) == 0:
1315             self.master = currentGitBranch()
1316             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1317                 die("Detecting current git branch failed!")
1318         elif len(args) == 1:
1319             self.master = args[0]
1320             if not branchExists(self.master):
1321                 die("Branch %s does not exist" % self.master)
1322         else:
1323             return False
1324
1325         allowSubmit = gitConfig("git-p4.allowSubmit")
1326         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1327             die("%s is not in git-p4.allowSubmit" % self.master)
1328
1329         [upstream, settings] = findUpstreamBranchPoint()
1330         self.depotPath = settings['depot-paths'][0]
1331         if len(self.origin) == 0:
1332             self.origin = upstream
1333
1334         if self.preserveUser:
1335             if not self.canChangeChangelists():
1336                 die("Cannot preserve user names without p4 super-user or admin permissions")
1337
1338         if self.verbose:
1339             print "Origin branch is " + self.origin
1340
1341         if len(self.depotPath) == 0:
1342             print "Internal error: cannot locate perforce depot path from existing branches"
1343             sys.exit(128)
1344
1345         self.useClientSpec = False
1346         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1347             self.useClientSpec = True
1348         if self.useClientSpec:
1349             self.clientSpecDirs = getClientSpec()
1350
1351         if self.useClientSpec:
1352             # all files are relative to the client spec
1353             self.clientPath = getClientRoot()
1354         else:
1355             self.clientPath = p4Where(self.depotPath)
1356
1357         if self.clientPath == "":
1358             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1359
1360         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1361         self.oldWorkingDirectory = os.getcwd()
1362
1363         # ensure the clientPath exists
1364         if not os.path.exists(self.clientPath):
1365             os.makedirs(self.clientPath)
1366
1367         chdir(self.clientPath)
1368         print "Synchronizing p4 checkout..."
1369         p4_sync("...")
1370         self.check()
1371
1372         commits = []
1373         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1374             commits.append(line.strip())
1375         commits.reverse()
1376
1377         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1378             self.checkAuthorship = False
1379         else:
1380             self.checkAuthorship = True
1381
1382         if self.preserveUser:
1383             self.checkValidP4Users(commits)
1384
1385         while len(commits) > 0:
1386             commit = commits[0]
1387             commits = commits[1:]
1388             self.applyCommit(commit)
1389             if not self.interactive:
1390                 break
1391
1392         if len(commits) == 0:
1393             print "All changes applied!"
1394             chdir(self.oldWorkingDirectory)
1395
1396             sync = P4Sync()
1397             sync.run([])
1398
1399             rebase = P4Rebase()
1400             rebase.rebase()
1401
1402         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1403             self.exportLabels = True
1404
1405         if self.exportLabels:
1406             p4Labels = getP4Labels(self.depotPath)
1407             gitTags = getGitTags()
1408
1409             missingGitTags = gitTags - p4Labels
1410             self.exportGitTags(missingGitTags)
1411
1412         return True
1413
1414 class View(object):
1415     """Represent a p4 view ("p4 help views"), and map files in a
1416        repo according to the view."""
1417
1418     class Path(object):
1419         """A depot or client path, possibly containing wildcards.
1420            The only one supported is ... at the end, currently.
1421            Initialize with the full path, with //depot or //client."""
1422
1423         def __init__(self, path, is_depot):
1424             self.path = path
1425             self.is_depot = is_depot
1426             self.find_wildcards()
1427             # remember the prefix bit, useful for relative mappings
1428             m = re.match("(//[^/]+/)", self.path)
1429             if not m:
1430                 die("Path %s does not start with //prefix/" % self.path)
1431             prefix = m.group(1)
1432             if not self.is_depot:
1433                 # strip //client/ on client paths
1434                 self.path = self.path[len(prefix):]
1435
1436         def find_wildcards(self):
1437             """Make sure wildcards are valid, and set up internal
1438                variables."""
1439
1440             self.ends_triple_dot = False
1441             # There are three wildcards allowed in p4 views
1442             # (see "p4 help views").  This code knows how to
1443             # handle "..." (only at the end), but cannot deal with
1444             # "%%n" or "*".  Only check the depot_side, as p4 should
1445             # validate that the client_side matches too.
1446             if re.search(r'%%[1-9]', self.path):
1447                 die("Can't handle %%n wildcards in view: %s" % self.path)
1448             if self.path.find("*") >= 0:
1449                 die("Can't handle * wildcards in view: %s" % self.path)
1450             triple_dot_index = self.path.find("...")
1451             if triple_dot_index >= 0:
1452                 if triple_dot_index != len(self.path) - 3:
1453                     die("Can handle only single ... wildcard, at end: %s" %
1454                         self.path)
1455                 self.ends_triple_dot = True
1456
1457         def ensure_compatible(self, other_path):
1458             """Make sure the wildcards agree."""
1459             if self.ends_triple_dot != other_path.ends_triple_dot:
1460                  die("Both paths must end with ... if either does;\n" +
1461                      "paths: %s %s" % (self.path, other_path.path))
1462
1463         def match_wildcards(self, test_path):
1464             """See if this test_path matches us, and fill in the value
1465                of the wildcards if so.  Returns a tuple of
1466                (True|False, wildcards[]).  For now, only the ... at end
1467                is supported, so at most one wildcard."""
1468             if self.ends_triple_dot:
1469                 dotless = self.path[:-3]
1470                 if test_path.startswith(dotless):
1471                     wildcard = test_path[len(dotless):]
1472                     return (True, [ wildcard ])
1473             else:
1474                 if test_path == self.path:
1475                     return (True, [])
1476             return (False, [])
1477
1478         def match(self, test_path):
1479             """Just return if it matches; don't bother with the wildcards."""
1480             b, _ = self.match_wildcards(test_path)
1481             return b
1482
1483         def fill_in_wildcards(self, wildcards):
1484             """Return the relative path, with the wildcards filled in
1485                if there are any."""
1486             if self.ends_triple_dot:
1487                 return self.path[:-3] + wildcards[0]
1488             else:
1489                 return self.path
1490
1491     class Mapping(object):
1492         def __init__(self, depot_side, client_side, overlay, exclude):
1493             # depot_side is without the trailing /... if it had one
1494             self.depot_side = View.Path(depot_side, is_depot=True)
1495             self.client_side = View.Path(client_side, is_depot=False)
1496             self.overlay = overlay  # started with "+"
1497             self.exclude = exclude  # started with "-"
1498             assert not (self.overlay and self.exclude)
1499             self.depot_side.ensure_compatible(self.client_side)
1500
1501         def __str__(self):
1502             c = " "
1503             if self.overlay:
1504                 c = "+"
1505             if self.exclude:
1506                 c = "-"
1507             return "View.Mapping: %s%s -> %s" % \
1508                    (c, self.depot_side.path, self.client_side.path)
1509
1510         def map_depot_to_client(self, depot_path):
1511             """Calculate the client path if using this mapping on the
1512                given depot path; does not consider the effect of other
1513                mappings in a view.  Even excluded mappings are returned."""
1514             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1515             if not matches:
1516                 return ""
1517             client_path = self.client_side.fill_in_wildcards(wildcards)
1518             return client_path
1519
1520     #
1521     # View methods
1522     #
1523     def __init__(self):
1524         self.mappings = []
1525
1526     def append(self, view_line):
1527         """Parse a view line, splitting it into depot and client
1528            sides.  Append to self.mappings, preserving order."""
1529
1530         # Split the view line into exactly two words.  P4 enforces
1531         # structure on these lines that simplifies this quite a bit.
1532         #
1533         # Either or both words may be double-quoted.
1534         # Single quotes do not matter.
1535         # Double-quote marks cannot occur inside the words.
1536         # A + or - prefix is also inside the quotes.
1537         # There are no quotes unless they contain a space.
1538         # The line is already white-space stripped.
1539         # The two words are separated by a single space.
1540         #
1541         if view_line[0] == '"':
1542             # First word is double quoted.  Find its end.
1543             close_quote_index = view_line.find('"', 1)
1544             if close_quote_index <= 0:
1545                 die("No first-word closing quote found: %s" % view_line)
1546             depot_side = view_line[1:close_quote_index]
1547             # skip closing quote and space
1548             rhs_index = close_quote_index + 1 + 1
1549         else:
1550             space_index = view_line.find(" ")
1551             if space_index <= 0:
1552                 die("No word-splitting space found: %s" % view_line)
1553             depot_side = view_line[0:space_index]
1554             rhs_index = space_index + 1
1555
1556         if view_line[rhs_index] == '"':
1557             # Second word is double quoted.  Make sure there is a
1558             # double quote at the end too.
1559             if not view_line.endswith('"'):
1560                 die("View line with rhs quote should end with one: %s" %
1561                     view_line)
1562             # skip the quotes
1563             client_side = view_line[rhs_index+1:-1]
1564         else:
1565             client_side = view_line[rhs_index:]
1566
1567         # prefix + means overlay on previous mapping
1568         overlay = False
1569         if depot_side.startswith("+"):
1570             overlay = True
1571             depot_side = depot_side[1:]
1572
1573         # prefix - means exclude this path
1574         exclude = False
1575         if depot_side.startswith("-"):
1576             exclude = True
1577             depot_side = depot_side[1:]
1578
1579         m = View.Mapping(depot_side, client_side, overlay, exclude)
1580         self.mappings.append(m)
1581
1582     def map_in_client(self, depot_path):
1583         """Return the relative location in the client where this
1584            depot file should live.  Returns "" if the file should
1585            not be mapped in the client."""
1586
1587         paths_filled = []
1588         client_path = ""
1589
1590         # look at later entries first
1591         for m in self.mappings[::-1]:
1592
1593             # see where will this path end up in the client
1594             p = m.map_depot_to_client(depot_path)
1595
1596             if p == "":
1597                 # Depot path does not belong in client.  Must remember
1598                 # this, as previous items should not cause files to
1599                 # exist in this path either.  Remember that the list is
1600                 # being walked from the end, which has higher precedence.
1601                 # Overlap mappings do not exclude previous mappings.
1602                 if not m.overlay:
1603                     paths_filled.append(m.client_side)
1604
1605             else:
1606                 # This mapping matched; no need to search any further.
1607                 # But, the mapping could be rejected if the client path
1608                 # has already been claimed by an earlier mapping (i.e.
1609                 # one later in the list, which we are walking backwards).
1610                 already_mapped_in_client = False
1611                 for f in paths_filled:
1612                     # this is View.Path.match
1613                     if f.match(p):
1614                         already_mapped_in_client = True
1615                         break
1616                 if not already_mapped_in_client:
1617                     # Include this file, unless it is from a line that
1618                     # explicitly said to exclude it.
1619                     if not m.exclude:
1620                         client_path = p
1621
1622                 # a match, even if rejected, always stops the search
1623                 break
1624
1625         return client_path
1626
1627 class P4Sync(Command, P4UserMap):
1628     delete_actions = ( "delete", "move/delete", "purge" )
1629
1630     def __init__(self):
1631         Command.__init__(self)
1632         P4UserMap.__init__(self)
1633         self.options = [
1634                 optparse.make_option("--branch", dest="branch"),
1635                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1636                 optparse.make_option("--changesfile", dest="changesFile"),
1637                 optparse.make_option("--silent", dest="silent", action="store_true"),
1638                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1639                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1640                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1641                                      help="Import into refs/heads/ , not refs/remotes"),
1642                 optparse.make_option("--max-changes", dest="maxChanges"),
1643                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1644                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1645                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1646                                      help="Only sync files that are included in the Perforce Client Spec")
1647         ]
1648         self.description = """Imports from Perforce into a git repository.\n
1649     example:
1650     //depot/my/project/ -- to import the current head
1651     //depot/my/project/@all -- to import everything
1652     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1653
1654     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1655
1656         self.usage += " //depot/path[@revRange]"
1657         self.silent = False
1658         self.createdBranches = set()
1659         self.committedChanges = set()
1660         self.branch = ""
1661         self.detectBranches = False
1662         self.detectLabels = False
1663         self.importLabels = False
1664         self.changesFile = ""
1665         self.syncWithOrigin = True
1666         self.importIntoRemotes = True
1667         self.maxChanges = ""
1668         self.isWindows = (platform.system() == "Windows")
1669         self.keepRepoPath = False
1670         self.depotPaths = None
1671         self.p4BranchesInGit = []
1672         self.cloneExclude = []
1673         self.useClientSpec = False
1674         self.useClientSpec_from_options = False
1675         self.clientSpecDirs = None
1676         self.tempBranches = []
1677         self.tempBranchLocation = "git-p4-tmp"
1678
1679         if gitConfig("git-p4.syncFromOrigin") == "false":
1680             self.syncWithOrigin = False
1681
1682     #
1683     # P4 wildcards are not allowed in filenames.  P4 complains
1684     # if you simply add them, but you can force it with "-f", in
1685     # which case it translates them into %xx encoding internally.
1686     # Search for and fix just these four characters.  Do % last so
1687     # that fixing it does not inadvertently create new %-escapes.
1688     #
1689     def wildcard_decode(self, path):
1690         # Cannot have * in a filename in windows; untested as to
1691         # what p4 would do in such a case.
1692         if not self.isWindows:
1693             path = path.replace("%2A", "*")
1694         path = path.replace("%23", "#") \
1695                    .replace("%40", "@") \
1696                    .replace("%25", "%")
1697         return path
1698
1699     # Force a checkpoint in fast-import and wait for it to finish
1700     def checkpoint(self):
1701         self.gitStream.write("checkpoint\n\n")
1702         self.gitStream.write("progress checkpoint\n\n")
1703         out = self.gitOutput.readline()
1704         if self.verbose:
1705             print "checkpoint finished: " + out
1706
1707     def extractFilesFromCommit(self, commit):
1708         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1709                              for path in self.cloneExclude]
1710         files = []
1711         fnum = 0
1712         while commit.has_key("depotFile%s" % fnum):
1713             path =  commit["depotFile%s" % fnum]
1714
1715             if [p for p in self.cloneExclude
1716                 if p4PathStartsWith(path, p)]:
1717                 found = False
1718             else:
1719                 found = [p for p in self.depotPaths
1720                          if p4PathStartsWith(path, p)]
1721             if not found:
1722                 fnum = fnum + 1
1723                 continue
1724
1725             file = {}
1726             file["path"] = path
1727             file["rev"] = commit["rev%s" % fnum]
1728             file["action"] = commit["action%s" % fnum]
1729             file["type"] = commit["type%s" % fnum]
1730             files.append(file)
1731             fnum = fnum + 1
1732         return files
1733
1734     def stripRepoPath(self, path, prefixes):
1735         if self.useClientSpec:
1736             return self.clientSpecDirs.map_in_client(path)
1737
1738         if self.keepRepoPath:
1739             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1740
1741         for p in prefixes:
1742             if p4PathStartsWith(path, p):
1743                 path = path[len(p):]
1744
1745         return path
1746
1747     def splitFilesIntoBranches(self, commit):
1748         branches = {}
1749         fnum = 0
1750         while commit.has_key("depotFile%s" % fnum):
1751             path =  commit["depotFile%s" % fnum]
1752             found = [p for p in self.depotPaths
1753                      if p4PathStartsWith(path, p)]
1754             if not found:
1755                 fnum = fnum + 1
1756                 continue
1757
1758             file = {}
1759             file["path"] = path
1760             file["rev"] = commit["rev%s" % fnum]
1761             file["action"] = commit["action%s" % fnum]
1762             file["type"] = commit["type%s" % fnum]
1763             fnum = fnum + 1
1764
1765             relPath = self.stripRepoPath(path, self.depotPaths)
1766
1767             for branch in self.knownBranches.keys():
1768
1769                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1770                 if relPath.startswith(branch + "/"):
1771                     if branch not in branches:
1772                         branches[branch] = []
1773                     branches[branch].append(file)
1774                     break
1775
1776         return branches
1777
1778     # output one file from the P4 stream
1779     # - helper for streamP4Files
1780
1781     def streamOneP4File(self, file, contents):
1782         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1783         relPath = self.wildcard_decode(relPath)
1784         if verbose:
1785             sys.stderr.write("%s\n" % relPath)
1786
1787         (type_base, type_mods) = split_p4_type(file["type"])
1788
1789         git_mode = "100644"
1790         if "x" in type_mods:
1791             git_mode = "100755"
1792         if type_base == "symlink":
1793             git_mode = "120000"
1794             # p4 print on a symlink contains "target\n"; remove the newline
1795             data = ''.join(contents)
1796             contents = [data[:-1]]
1797
1798         if type_base == "utf16":
1799             # p4 delivers different text in the python output to -G
1800             # than it does when using "print -o", or normal p4 client
1801             # operations.  utf16 is converted to ascii or utf8, perhaps.
1802             # But ascii text saved as -t utf16 is completely mangled.
1803             # Invoke print -o to get the real contents.
1804             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1805             contents = [ text ]
1806
1807         if type_base == "apple":
1808             # Apple filetype files will be streamed as a concatenation of
1809             # its appledouble header and the contents.  This is useless
1810             # on both macs and non-macs.  If using "print -q -o xx", it
1811             # will create "xx" with the data, and "%xx" with the header.
1812             # This is also not very useful.
1813             #
1814             # Ideally, someday, this script can learn how to generate
1815             # appledouble files directly and import those to git, but
1816             # non-mac machines can never find a use for apple filetype.
1817             print "\nIgnoring apple filetype file %s" % file['depotFile']
1818             return
1819
1820         # Perhaps windows wants unicode, utf16 newlines translated too;
1821         # but this is not doing it.
1822         if self.isWindows and type_base == "text":
1823             mangled = []
1824             for data in contents:
1825                 data = data.replace("\r\n", "\n")
1826                 mangled.append(data)
1827             contents = mangled
1828
1829         # Note that we do not try to de-mangle keywords on utf16 files,
1830         # even though in theory somebody may want that.
1831         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1832         if pattern:
1833             regexp = re.compile(pattern, re.VERBOSE)
1834             text = ''.join(contents)
1835             text = regexp.sub(r'$\1$', text)
1836             contents = [ text ]
1837
1838         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1839
1840         # total length...
1841         length = 0
1842         for d in contents:
1843             length = length + len(d)
1844
1845         self.gitStream.write("data %d\n" % length)
1846         for d in contents:
1847             self.gitStream.write(d)
1848         self.gitStream.write("\n")
1849
1850     def streamOneP4Deletion(self, file):
1851         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1852         if verbose:
1853             sys.stderr.write("delete %s\n" % relPath)
1854         self.gitStream.write("D %s\n" % relPath)
1855
1856     # handle another chunk of streaming data
1857     def streamP4FilesCb(self, marshalled):
1858
1859         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1860             # start of a new file - output the old one first
1861             self.streamOneP4File(self.stream_file, self.stream_contents)
1862             self.stream_file = {}
1863             self.stream_contents = []
1864             self.stream_have_file_info = False
1865
1866         # pick up the new file information... for the
1867         # 'data' field we need to append to our array
1868         for k in marshalled.keys():
1869             if k == 'data':
1870                 self.stream_contents.append(marshalled['data'])
1871             else:
1872                 self.stream_file[k] = marshalled[k]
1873
1874         self.stream_have_file_info = True
1875
1876     # Stream directly from "p4 files" into "git fast-import"
1877     def streamP4Files(self, files):
1878         filesForCommit = []
1879         filesToRead = []
1880         filesToDelete = []
1881
1882         for f in files:
1883             # if using a client spec, only add the files that have
1884             # a path in the client
1885             if self.clientSpecDirs:
1886                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1887                     continue
1888
1889             filesForCommit.append(f)
1890             if f['action'] in self.delete_actions:
1891                 filesToDelete.append(f)
1892             else:
1893                 filesToRead.append(f)
1894
1895         # deleted files...
1896         for f in filesToDelete:
1897             self.streamOneP4Deletion(f)
1898
1899         if len(filesToRead) > 0:
1900             self.stream_file = {}
1901             self.stream_contents = []
1902             self.stream_have_file_info = False
1903
1904             # curry self argument
1905             def streamP4FilesCbSelf(entry):
1906                 self.streamP4FilesCb(entry)
1907
1908             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1909
1910             p4CmdList(["-x", "-", "print"],
1911                       stdin=fileArgs,
1912                       cb=streamP4FilesCbSelf)
1913
1914             # do the last chunk
1915             if self.stream_file.has_key('depotFile'):
1916                 self.streamOneP4File(self.stream_file, self.stream_contents)
1917
1918     def make_email(self, userid):
1919         if userid in self.users:
1920             return self.users[userid]
1921         else:
1922             return "%s <a@b>" % userid
1923
1924     # Stream a p4 tag
1925     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
1926         if verbose:
1927             print "writing tag %s for commit %s" % (labelName, commit)
1928         gitStream.write("tag %s\n" % labelName)
1929         gitStream.write("from %s\n" % commit)
1930
1931         if labelDetails.has_key('Owner'):
1932             owner = labelDetails["Owner"]
1933         else:
1934             owner = None
1935
1936         # Try to use the owner of the p4 label, or failing that,
1937         # the current p4 user id.
1938         if owner:
1939             email = self.make_email(owner)
1940         else:
1941             email = self.make_email(self.p4UserId())
1942         tagger = "%s %s %s" % (email, epoch, self.tz)
1943
1944         gitStream.write("tagger %s\n" % tagger)
1945
1946         print "labelDetails=",labelDetails
1947         if labelDetails.has_key('Description'):
1948             description = labelDetails['Description']
1949         else:
1950             description = 'Label from git p4'
1951
1952         gitStream.write("data %d\n" % len(description))
1953         gitStream.write(description)
1954         gitStream.write("\n")
1955
1956     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1957         epoch = details["time"]
1958         author = details["user"]
1959         self.branchPrefixes = branchPrefixes
1960
1961         if self.verbose:
1962             print "commit into %s" % branch
1963
1964         # start with reading files; if that fails, we should not
1965         # create a commit.
1966         new_files = []
1967         for f in files:
1968             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1969                 new_files.append (f)
1970             else:
1971                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1972
1973         self.gitStream.write("commit %s\n" % branch)
1974 #        gitStream.write("mark :%s\n" % details["change"])
1975         self.committedChanges.add(int(details["change"]))
1976         committer = ""
1977         if author not in self.users:
1978             self.getUserMapFromPerforceServer()
1979         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
1980
1981         self.gitStream.write("committer %s\n" % committer)
1982
1983         self.gitStream.write("data <<EOT\n")
1984         self.gitStream.write(details["desc"])
1985         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1986                              % (','.join (branchPrefixes), details["change"]))
1987         if len(details['options']) > 0:
1988             self.gitStream.write(": options = %s" % details['options'])
1989         self.gitStream.write("]\nEOT\n\n")
1990
1991         if len(parent) > 0:
1992             if self.verbose:
1993                 print "parent %s" % parent
1994             self.gitStream.write("from %s\n" % parent)
1995
1996         self.streamP4Files(new_files)
1997         self.gitStream.write("\n")
1998
1999         change = int(details["change"])
2000
2001         if self.labels.has_key(change):
2002             label = self.labels[change]
2003             labelDetails = label[0]
2004             labelRevisions = label[1]
2005             if self.verbose:
2006                 print "Change %s is labelled %s" % (change, labelDetails)
2007
2008             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2009                                                     for p in branchPrefixes])
2010
2011             if len(files) == len(labelRevisions):
2012
2013                 cleanedFiles = {}
2014                 for info in files:
2015                     if info["action"] in self.delete_actions:
2016                         continue
2017                     cleanedFiles[info["depotFile"]] = info["rev"]
2018
2019                 if cleanedFiles == labelRevisions:
2020                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2021
2022                 else:
2023                     if not self.silent:
2024                         print ("Tag %s does not match with change %s: files do not match."
2025                                % (labelDetails["label"], change))
2026
2027             else:
2028                 if not self.silent:
2029                     print ("Tag %s does not match with change %s: file count is different."
2030                            % (labelDetails["label"], change))
2031
2032     # Build a dictionary of changelists and labels, for "detect-labels" option.
2033     def getLabels(self):
2034         self.labels = {}
2035
2036         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2037         if len(l) > 0 and not self.silent:
2038             print "Finding files belonging to labels in %s" % `self.depotPaths`
2039
2040         for output in l:
2041             label = output["label"]
2042             revisions = {}
2043             newestChange = 0
2044             if self.verbose:
2045                 print "Querying files for label %s" % label
2046             for file in p4CmdList(["files"] +
2047                                       ["%s...@%s" % (p, label)
2048                                           for p in self.depotPaths]):
2049                 revisions[file["depotFile"]] = file["rev"]
2050                 change = int(file["change"])
2051                 if change > newestChange:
2052                     newestChange = change
2053
2054             self.labels[newestChange] = [output, revisions]
2055
2056         if self.verbose:
2057             print "Label changes: %s" % self.labels.keys()
2058
2059     # Import p4 labels as git tags. A direct mapping does not
2060     # exist, so assume that if all the files are at the same revision
2061     # then we can use that, or it's something more complicated we should
2062     # just ignore.
2063     def importP4Labels(self, stream, p4Labels):
2064         if verbose:
2065             print "import p4 labels: " + ' '.join(p4Labels)
2066
2067         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2068         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2069         if len(validLabelRegexp) == 0:
2070             validLabelRegexp = defaultLabelRegexp
2071         m = re.compile(validLabelRegexp)
2072
2073         for name in p4Labels:
2074             commitFound = False
2075
2076             if not m.match(name):
2077                 if verbose:
2078                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2079                 continue
2080
2081             if name in ignoredP4Labels:
2082                 continue
2083
2084             labelDetails = p4CmdList(['label', "-o", name])[0]
2085
2086             # get the most recent changelist for each file in this label
2087             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2088                                 for p in self.depotPaths])
2089
2090             if change.has_key('change'):
2091                 # find the corresponding git commit; take the oldest commit
2092                 changelist = int(change['change'])
2093                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2094                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2095                 if len(gitCommit) == 0:
2096                     print "could not find git commit for changelist %d" % changelist
2097                 else:
2098                     gitCommit = gitCommit.strip()
2099                     commitFound = True
2100                     # Convert from p4 time format
2101                     try:
2102                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2103                     except ValueError:
2104                         print "Could not convert label time %s" % labelDetail['Update']
2105                         tmwhen = 1
2106
2107                     when = int(time.mktime(tmwhen))
2108                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2109                     if verbose:
2110                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2111             else:
2112                 if verbose:
2113                     print "Label %s has no changelists - possibly deleted?" % name
2114
2115             if not commitFound:
2116                 # We can't import this label; don't try again as it will get very
2117                 # expensive repeatedly fetching all the files for labels that will
2118                 # never be imported. If the label is moved in the future, the
2119                 # ignore will need to be removed manually.
2120                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2121
2122     def guessProjectName(self):
2123         for p in self.depotPaths:
2124             if p.endswith("/"):
2125                 p = p[:-1]
2126             p = p[p.strip().rfind("/") + 1:]
2127             if not p.endswith("/"):
2128                p += "/"
2129             return p
2130
2131     def getBranchMapping(self):
2132         lostAndFoundBranches = set()
2133
2134         user = gitConfig("git-p4.branchUser")
2135         if len(user) > 0:
2136             command = "branches -u %s" % user
2137         else:
2138             command = "branches"
2139
2140         for info in p4CmdList(command):
2141             details = p4Cmd(["branch", "-o", info["branch"]])
2142             viewIdx = 0
2143             while details.has_key("View%s" % viewIdx):
2144                 paths = details["View%s" % viewIdx].split(" ")
2145                 viewIdx = viewIdx + 1
2146                 # require standard //depot/foo/... //depot/bar/... mapping
2147                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2148                     continue
2149                 source = paths[0]
2150                 destination = paths[1]
2151                 ## HACK
2152                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2153                     source = source[len(self.depotPaths[0]):-4]
2154                     destination = destination[len(self.depotPaths[0]):-4]
2155
2156                     if destination in self.knownBranches:
2157                         if not self.silent:
2158                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2159                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2160                         continue
2161
2162                     self.knownBranches[destination] = source
2163
2164                     lostAndFoundBranches.discard(destination)
2165
2166                     if source not in self.knownBranches:
2167                         lostAndFoundBranches.add(source)
2168
2169         # Perforce does not strictly require branches to be defined, so we also
2170         # check git config for a branch list.
2171         #
2172         # Example of branch definition in git config file:
2173         # [git-p4]
2174         #   branchList=main:branchA
2175         #   branchList=main:branchB
2176         #   branchList=branchA:branchC
2177         configBranches = gitConfigList("git-p4.branchList")
2178         for branch in configBranches:
2179             if branch:
2180                 (source, destination) = branch.split(":")
2181                 self.knownBranches[destination] = source
2182
2183                 lostAndFoundBranches.discard(destination)
2184
2185                 if source not in self.knownBranches:
2186                     lostAndFoundBranches.add(source)
2187
2188
2189         for branch in lostAndFoundBranches:
2190             self.knownBranches[branch] = branch
2191
2192     def getBranchMappingFromGitBranches(self):
2193         branches = p4BranchesInGit(self.importIntoRemotes)
2194         for branch in branches.keys():
2195             if branch == "master":
2196                 branch = "main"
2197             else:
2198                 branch = branch[len(self.projectName):]
2199             self.knownBranches[branch] = branch
2200
2201     def listExistingP4GitBranches(self):
2202         # branches holds mapping from name to commit
2203         branches = p4BranchesInGit(self.importIntoRemotes)
2204         self.p4BranchesInGit = branches.keys()
2205         for branch in branches.keys():
2206             self.initialParents[self.refPrefix + branch] = branches[branch]
2207
2208     def updateOptionDict(self, d):
2209         option_keys = {}
2210         if self.keepRepoPath:
2211             option_keys['keepRepoPath'] = 1
2212
2213         d["options"] = ' '.join(sorted(option_keys.keys()))
2214
2215     def readOptions(self, d):
2216         self.keepRepoPath = (d.has_key('options')
2217                              and ('keepRepoPath' in d['options']))
2218
2219     def gitRefForBranch(self, branch):
2220         if branch == "main":
2221             return self.refPrefix + "master"
2222
2223         if len(branch) <= 0:
2224             return branch
2225
2226         return self.refPrefix + self.projectName + branch
2227
2228     def gitCommitByP4Change(self, ref, change):
2229         if self.verbose:
2230             print "looking in ref " + ref + " for change %s using bisect..." % change
2231
2232         earliestCommit = ""
2233         latestCommit = parseRevision(ref)
2234
2235         while True:
2236             if self.verbose:
2237                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2238             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2239             if len(next) == 0:
2240                 if self.verbose:
2241                     print "argh"
2242                 return ""
2243             log = extractLogMessageFromGitCommit(next)
2244             settings = extractSettingsGitLog(log)
2245             currentChange = int(settings['change'])
2246             if self.verbose:
2247                 print "current change %s" % currentChange
2248
2249             if currentChange == change:
2250                 if self.verbose:
2251                     print "found %s" % next
2252                 return next
2253
2254             if currentChange < change:
2255                 earliestCommit = "^%s" % next
2256             else:
2257                 latestCommit = "%s" % next
2258
2259         return ""
2260
2261     def importNewBranch(self, branch, maxChange):
2262         # make fast-import flush all changes to disk and update the refs using the checkpoint
2263         # command so that we can try to find the branch parent in the git history
2264         self.gitStream.write("checkpoint\n\n");
2265         self.gitStream.flush();
2266         branchPrefix = self.depotPaths[0] + branch + "/"
2267         range = "@1,%s" % maxChange
2268         #print "prefix" + branchPrefix
2269         changes = p4ChangesForPaths([branchPrefix], range)
2270         if len(changes) <= 0:
2271             return False
2272         firstChange = changes[0]
2273         #print "first change in branch: %s" % firstChange
2274         sourceBranch = self.knownBranches[branch]
2275         sourceDepotPath = self.depotPaths[0] + sourceBranch
2276         sourceRef = self.gitRefForBranch(sourceBranch)
2277         #print "source " + sourceBranch
2278
2279         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2280         #print "branch parent: %s" % branchParentChange
2281         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2282         if len(gitParent) > 0:
2283             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2284             #print "parent git commit: %s" % gitParent
2285
2286         self.importChanges(changes)
2287         return True
2288
2289     def searchParent(self, parent, branch, target):
2290         parentFound = False
2291         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2292             blob = blob.strip()
2293             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2294                 parentFound = True
2295                 if self.verbose:
2296                     print "Found parent of %s in commit %s" % (branch, blob)
2297                 break
2298         if parentFound:
2299             return blob
2300         else:
2301             return None
2302
2303     def importChanges(self, changes):
2304         cnt = 1
2305         for change in changes:
2306             description = p4Cmd(["describe", str(change)])
2307             self.updateOptionDict(description)
2308
2309             if not self.silent:
2310                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2311                 sys.stdout.flush()
2312             cnt = cnt + 1
2313
2314             try:
2315                 if self.detectBranches:
2316                     branches = self.splitFilesIntoBranches(description)
2317                     for branch in branches.keys():
2318                         ## HACK  --hwn
2319                         branchPrefix = self.depotPaths[0] + branch + "/"
2320
2321                         parent = ""
2322
2323                         filesForCommit = branches[branch]
2324
2325                         if self.verbose:
2326                             print "branch is %s" % branch
2327
2328                         self.updatedBranches.add(branch)
2329
2330                         if branch not in self.createdBranches:
2331                             self.createdBranches.add(branch)
2332                             parent = self.knownBranches[branch]
2333                             if parent == branch:
2334                                 parent = ""
2335                             else:
2336                                 fullBranch = self.projectName + branch
2337                                 if fullBranch not in self.p4BranchesInGit:
2338                                     if not self.silent:
2339                                         print("\n    Importing new branch %s" % fullBranch);
2340                                     if self.importNewBranch(branch, change - 1):
2341                                         parent = ""
2342                                         self.p4BranchesInGit.append(fullBranch)
2343                                     if not self.silent:
2344                                         print("\n    Resuming with change %s" % change);
2345
2346                                 if self.verbose:
2347                                     print "parent determined through known branches: %s" % parent
2348
2349                         branch = self.gitRefForBranch(branch)
2350                         parent = self.gitRefForBranch(parent)
2351
2352                         if self.verbose:
2353                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2354
2355                         if len(parent) == 0 and branch in self.initialParents:
2356                             parent = self.initialParents[branch]
2357                             del self.initialParents[branch]
2358
2359                         blob = None
2360                         if len(parent) > 0:
2361                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2362                             if self.verbose:
2363                                 print "Creating temporary branch: " + tempBranch
2364                             self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2365                             self.tempBranches.append(tempBranch)
2366                             self.checkpoint()
2367                             blob = self.searchParent(parent, branch, tempBranch)
2368                         if blob:
2369                             self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2370                         else:
2371                             if self.verbose:
2372                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2373                             self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2374                 else:
2375                     files = self.extractFilesFromCommit(description)
2376                     self.commit(description, files, self.branch, self.depotPaths,
2377                                 self.initialParent)
2378                     self.initialParent = ""
2379             except IOError:
2380                 print self.gitError.read()
2381                 sys.exit(1)
2382
2383     def importHeadRevision(self, revision):
2384         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2385
2386         details = {}
2387         details["user"] = "git perforce import user"
2388         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2389                            % (' '.join(self.depotPaths), revision))
2390         details["change"] = revision
2391         newestRevision = 0
2392
2393         fileCnt = 0
2394         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2395
2396         for info in p4CmdList(["files"] + fileArgs):
2397
2398             if 'code' in info and info['code'] == 'error':
2399                 sys.stderr.write("p4 returned an error: %s\n"
2400                                  % info['data'])
2401                 if info['data'].find("must refer to client") >= 0:
2402                     sys.stderr.write("This particular p4 error is misleading.\n")
2403                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2404                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2405                 sys.exit(1)
2406             if 'p4ExitCode' in info:
2407                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2408                 sys.exit(1)
2409
2410
2411             change = int(info["change"])
2412             if change > newestRevision:
2413                 newestRevision = change
2414
2415             if info["action"] in self.delete_actions:
2416                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2417                 #fileCnt = fileCnt + 1
2418                 continue
2419
2420             for prop in ["depotFile", "rev", "action", "type" ]:
2421                 details["%s%s" % (prop, fileCnt)] = info[prop]
2422
2423             fileCnt = fileCnt + 1
2424
2425         details["change"] = newestRevision
2426
2427         # Use time from top-most change so that all git p4 clones of
2428         # the same p4 repo have the same commit SHA1s.
2429         res = p4CmdList("describe -s %d" % newestRevision)
2430         newestTime = None
2431         for r in res:
2432             if r.has_key('time'):
2433                 newestTime = int(r['time'])
2434         if newestTime is None:
2435             die("\"describe -s\" on newest change %d did not give a time")
2436         details["time"] = newestTime
2437
2438         self.updateOptionDict(details)
2439         try:
2440             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2441         except IOError:
2442             print "IO error with git fast-import. Is your git version recent enough?"
2443             print self.gitError.read()
2444
2445
2446     def run(self, args):
2447         self.depotPaths = []
2448         self.changeRange = ""
2449         self.initialParent = ""
2450         self.previousDepotPaths = []
2451
2452         # map from branch depot path to parent branch
2453         self.knownBranches = {}
2454         self.initialParents = {}
2455         self.hasOrigin = originP4BranchesExist()
2456         if not self.syncWithOrigin:
2457             self.hasOrigin = False
2458
2459         if self.importIntoRemotes:
2460             self.refPrefix = "refs/remotes/p4/"
2461         else:
2462             self.refPrefix = "refs/heads/p4/"
2463
2464         if self.syncWithOrigin and self.hasOrigin:
2465             if not self.silent:
2466                 print "Syncing with origin first by calling git fetch origin"
2467             system("git fetch origin")
2468
2469         if len(self.branch) == 0:
2470             self.branch = self.refPrefix + "master"
2471             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2472                 system("git update-ref %s refs/heads/p4" % self.branch)
2473                 system("git branch -D p4");
2474             # create it /after/ importing, when master exists
2475             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2476                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2477
2478         # accept either the command-line option, or the configuration variable
2479         if self.useClientSpec:
2480             # will use this after clone to set the variable
2481             self.useClientSpec_from_options = True
2482         else:
2483             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2484                 self.useClientSpec = True
2485         if self.useClientSpec:
2486             self.clientSpecDirs = getClientSpec()
2487
2488         # TODO: should always look at previous commits,
2489         # merge with previous imports, if possible.
2490         if args == []:
2491             if self.hasOrigin:
2492                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2493             self.listExistingP4GitBranches()
2494
2495             if len(self.p4BranchesInGit) > 1:
2496                 if not self.silent:
2497                     print "Importing from/into multiple branches"
2498                 self.detectBranches = True
2499
2500             if self.verbose:
2501                 print "branches: %s" % self.p4BranchesInGit
2502
2503             p4Change = 0
2504             for branch in self.p4BranchesInGit:
2505                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2506
2507                 settings = extractSettingsGitLog(logMsg)
2508
2509                 self.readOptions(settings)
2510                 if (settings.has_key('depot-paths')
2511                     and settings.has_key ('change')):
2512                     change = int(settings['change']) + 1
2513                     p4Change = max(p4Change, change)
2514
2515                     depotPaths = sorted(settings['depot-paths'])
2516                     if self.previousDepotPaths == []:
2517                         self.previousDepotPaths = depotPaths
2518                     else:
2519                         paths = []
2520                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2521                             prev_list = prev.split("/")
2522                             cur_list = cur.split("/")
2523                             for i in range(0, min(len(cur_list), len(prev_list))):
2524                                 if cur_list[i] <> prev_list[i]:
2525                                     i = i - 1
2526                                     break
2527
2528                             paths.append ("/".join(cur_list[:i + 1]))
2529
2530                         self.previousDepotPaths = paths
2531
2532             if p4Change > 0:
2533                 self.depotPaths = sorted(self.previousDepotPaths)
2534                 self.changeRange = "@%s,#head" % p4Change
2535                 if not self.detectBranches:
2536                     self.initialParent = parseRevision(self.branch)
2537                 if not self.silent and not self.detectBranches:
2538                     print "Performing incremental import into %s git branch" % self.branch
2539
2540         if not self.branch.startswith("refs/"):
2541             self.branch = "refs/heads/" + self.branch
2542
2543         if len(args) == 0 and self.depotPaths:
2544             if not self.silent:
2545                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2546         else:
2547             if self.depotPaths and self.depotPaths != args:
2548                 print ("previous import used depot path %s and now %s was specified. "
2549                        "This doesn't work!" % (' '.join (self.depotPaths),
2550                                                ' '.join (args)))
2551                 sys.exit(1)
2552
2553             self.depotPaths = sorted(args)
2554
2555         revision = ""
2556         self.users = {}
2557
2558         # Make sure no revision specifiers are used when --changesfile
2559         # is specified.
2560         bad_changesfile = False
2561         if len(self.changesFile) > 0:
2562             for p in self.depotPaths:
2563                 if p.find("@") >= 0 or p.find("#") >= 0:
2564                     bad_changesfile = True
2565                     break
2566         if bad_changesfile:
2567             die("Option --changesfile is incompatible with revision specifiers")
2568
2569         newPaths = []
2570         for p in self.depotPaths:
2571             if p.find("@") != -1:
2572                 atIdx = p.index("@")
2573                 self.changeRange = p[atIdx:]
2574                 if self.changeRange == "@all":
2575                     self.changeRange = ""
2576                 elif ',' not in self.changeRange:
2577                     revision = self.changeRange
2578                     self.changeRange = ""
2579                 p = p[:atIdx]
2580             elif p.find("#") != -1:
2581                 hashIdx = p.index("#")
2582                 revision = p[hashIdx:]
2583                 p = p[:hashIdx]
2584             elif self.previousDepotPaths == []:
2585                 # pay attention to changesfile, if given, else import
2586                 # the entire p4 tree at the head revision
2587                 if len(self.changesFile) == 0:
2588                     revision = "#head"
2589
2590             p = re.sub ("\.\.\.$", "", p)
2591             if not p.endswith("/"):
2592                 p += "/"
2593
2594             newPaths.append(p)
2595
2596         self.depotPaths = newPaths
2597
2598         self.loadUserMapFromCache()
2599         self.labels = {}
2600         if self.detectLabels:
2601             self.getLabels();
2602
2603         if self.detectBranches:
2604             ## FIXME - what's a P4 projectName ?
2605             self.projectName = self.guessProjectName()
2606
2607             if self.hasOrigin:
2608                 self.getBranchMappingFromGitBranches()
2609             else:
2610                 self.getBranchMapping()
2611             if self.verbose:
2612                 print "p4-git branches: %s" % self.p4BranchesInGit
2613                 print "initial parents: %s" % self.initialParents
2614             for b in self.p4BranchesInGit:
2615                 if b != "master":
2616
2617                     ## FIXME
2618                     b = b[len(self.projectName):]
2619                 self.createdBranches.add(b)
2620
2621         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2622
2623         importProcess = subprocess.Popen(["git", "fast-import"],
2624                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2625                                          stderr=subprocess.PIPE);
2626         self.gitOutput = importProcess.stdout
2627         self.gitStream = importProcess.stdin
2628         self.gitError = importProcess.stderr
2629
2630         if revision:
2631             self.importHeadRevision(revision)
2632         else:
2633             changes = []
2634
2635             if len(self.changesFile) > 0:
2636                 output = open(self.changesFile).readlines()
2637                 changeSet = set()
2638                 for line in output:
2639                     changeSet.add(int(line))
2640
2641                 for change in changeSet:
2642                     changes.append(change)
2643
2644                 changes.sort()
2645             else:
2646                 # catch "git p4 sync" with no new branches, in a repo that
2647                 # does not have any existing p4 branches
2648                 if len(args) == 0 and not self.p4BranchesInGit:
2649                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2650                 if self.verbose:
2651                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2652                                                               self.changeRange)
2653                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2654
2655                 if len(self.maxChanges) > 0:
2656                     changes = changes[:min(int(self.maxChanges), len(changes))]
2657
2658             if len(changes) == 0:
2659                 if not self.silent:
2660                     print "No changes to import!"
2661             else:
2662                 if not self.silent and not self.detectBranches:
2663                     print "Import destination: %s" % self.branch
2664
2665                 self.updatedBranches = set()
2666
2667                 self.importChanges(changes)
2668
2669                 if not self.silent:
2670                     print ""
2671                     if len(self.updatedBranches) > 0:
2672                         sys.stdout.write("Updated branches: ")
2673                         for b in self.updatedBranches:
2674                             sys.stdout.write("%s " % b)
2675                         sys.stdout.write("\n")
2676
2677         if gitConfig("git-p4.importLabels", "--bool") == "true":
2678             self.importLabels = True
2679
2680         if self.importLabels:
2681             p4Labels = getP4Labels(self.depotPaths)
2682             gitTags = getGitTags()
2683
2684             missingP4Labels = p4Labels - gitTags
2685             self.importP4Labels(self.gitStream, missingP4Labels)
2686
2687         self.gitStream.close()
2688         if importProcess.wait() != 0:
2689             die("fast-import failed: %s" % self.gitError.read())
2690         self.gitOutput.close()
2691         self.gitError.close()
2692
2693         # Cleanup temporary branches created during import
2694         if self.tempBranches != []:
2695             for branch in self.tempBranches:
2696                 read_pipe("git update-ref -d %s" % branch)
2697             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2698
2699         return True
2700
2701 class P4Rebase(Command):
2702     def __init__(self):
2703         Command.__init__(self)
2704         self.options = [
2705                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2706         ]
2707         self.importLabels = False
2708         self.description = ("Fetches the latest revision from perforce and "
2709                             + "rebases the current work (branch) against it")
2710
2711     def run(self, args):
2712         sync = P4Sync()
2713         sync.importLabels = self.importLabels
2714         sync.run([])
2715
2716         return self.rebase()
2717
2718     def rebase(self):
2719         if os.system("git update-index --refresh") != 0:
2720             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2721         if len(read_pipe("git diff-index HEAD --")) > 0:
2722             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2723
2724         [upstream, settings] = findUpstreamBranchPoint()
2725         if len(upstream) == 0:
2726             die("Cannot find upstream branchpoint for rebase")
2727
2728         # the branchpoint may be p4/foo~3, so strip off the parent
2729         upstream = re.sub("~[0-9]+$", "", upstream)
2730
2731         print "Rebasing the current branch onto %s" % upstream
2732         oldHead = read_pipe("git rev-parse HEAD").strip()
2733         system("git rebase %s" % upstream)
2734         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2735         return True
2736
2737 class P4Clone(P4Sync):
2738     def __init__(self):
2739         P4Sync.__init__(self)
2740         self.description = "Creates a new git repository and imports from Perforce into it"
2741         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2742         self.options += [
2743             optparse.make_option("--destination", dest="cloneDestination",
2744                                  action='store', default=None,
2745                                  help="where to leave result of the clone"),
2746             optparse.make_option("-/", dest="cloneExclude",
2747                                  action="append", type="string",
2748                                  help="exclude depot path"),
2749             optparse.make_option("--bare", dest="cloneBare",
2750                                  action="store_true", default=False),
2751         ]
2752         self.cloneDestination = None
2753         self.needsGit = False
2754         self.cloneBare = False
2755
2756     # This is required for the "append" cloneExclude action
2757     def ensure_value(self, attr, value):
2758         if not hasattr(self, attr) or getattr(self, attr) is None:
2759             setattr(self, attr, value)
2760         return getattr(self, attr)
2761
2762     def defaultDestination(self, args):
2763         ## TODO: use common prefix of args?
2764         depotPath = args[0]
2765         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2766         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2767         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2768         depotDir = re.sub(r"/$", "", depotDir)
2769         return os.path.split(depotDir)[1]
2770
2771     def run(self, args):
2772         if len(args) < 1:
2773             return False
2774
2775         if self.keepRepoPath and not self.cloneDestination:
2776             sys.stderr.write("Must specify destination for --keep-path\n")
2777             sys.exit(1)
2778
2779         depotPaths = args
2780
2781         if not self.cloneDestination and len(depotPaths) > 1:
2782             self.cloneDestination = depotPaths[-1]
2783             depotPaths = depotPaths[:-1]
2784
2785         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2786         for p in depotPaths:
2787             if not p.startswith("//"):
2788                 return False
2789
2790         if not self.cloneDestination:
2791             self.cloneDestination = self.defaultDestination(args)
2792
2793         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2794
2795         if not os.path.exists(self.cloneDestination):
2796             os.makedirs(self.cloneDestination)
2797         chdir(self.cloneDestination)
2798
2799         init_cmd = [ "git", "init" ]
2800         if self.cloneBare:
2801             init_cmd.append("--bare")
2802         subprocess.check_call(init_cmd)
2803
2804         if not P4Sync.run(self, depotPaths):
2805             return False
2806         if self.branch != "master":
2807             if self.importIntoRemotes:
2808                 masterbranch = "refs/remotes/p4/master"
2809             else:
2810                 masterbranch = "refs/heads/p4/master"
2811             if gitBranchExists(masterbranch):
2812                 system("git branch master %s" % masterbranch)
2813                 if not self.cloneBare:
2814                     system("git checkout -f")
2815             else:
2816                 print "Could not detect main branch. No checkout/master branch created."
2817
2818         # auto-set this variable if invoked with --use-client-spec
2819         if self.useClientSpec_from_options:
2820             system("git config --bool git-p4.useclientspec true")
2821
2822         return True
2823
2824 class P4Branches(Command):
2825     def __init__(self):
2826         Command.__init__(self)
2827         self.options = [ ]
2828         self.description = ("Shows the git branches that hold imports and their "
2829                             + "corresponding perforce depot paths")
2830         self.verbose = False
2831
2832     def run(self, args):
2833         if originP4BranchesExist():
2834             createOrUpdateBranchesFromOrigin()
2835
2836         cmdline = "git rev-parse --symbolic "
2837         cmdline += " --remotes"
2838
2839         for line in read_pipe_lines(cmdline):
2840             line = line.strip()
2841
2842             if not line.startswith('p4/') or line == "p4/HEAD":
2843                 continue
2844             branch = line
2845
2846             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2847             settings = extractSettingsGitLog(log)
2848
2849             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2850         return True
2851
2852 class HelpFormatter(optparse.IndentedHelpFormatter):
2853     def __init__(self):
2854         optparse.IndentedHelpFormatter.__init__(self)
2855
2856     def format_description(self, description):
2857         if description:
2858             return description + "\n"
2859         else:
2860             return ""
2861
2862 def printUsage(commands):
2863     print "usage: %s <command> [options]" % sys.argv[0]
2864     print ""
2865     print "valid commands: %s" % ", ".join(commands)
2866     print ""
2867     print "Try %s <command> --help for command specific help." % sys.argv[0]
2868     print ""
2869
2870 commands = {
2871     "debug" : P4Debug,
2872     "submit" : P4Submit,
2873     "commit" : P4Submit,
2874     "sync" : P4Sync,
2875     "rebase" : P4Rebase,
2876     "clone" : P4Clone,
2877     "rollback" : P4RollBack,
2878     "branches" : P4Branches
2879 }
2880
2881
2882 def main():
2883     if len(sys.argv[1:]) == 0:
2884         printUsage(commands.keys())
2885         sys.exit(2)
2886
2887     cmd = ""
2888     cmdName = sys.argv[1]
2889     try:
2890         klass = commands[cmdName]
2891         cmd = klass()
2892     except KeyError:
2893         print "unknown command %s" % cmdName
2894         print ""
2895         printUsage(commands.keys())
2896         sys.exit(2)
2897
2898     options = cmd.options
2899     cmd.gitdir = os.environ.get("GIT_DIR", None)
2900
2901     args = sys.argv[2:]
2902
2903     options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))
2904     if cmd.needsGit:
2905         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2906
2907     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2908                                    options,
2909                                    description = cmd.description,
2910                                    formatter = HelpFormatter())
2911
2912     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2913     global verbose
2914     verbose = cmd.verbose
2915     if cmd.needsGit:
2916         if cmd.gitdir == None:
2917             cmd.gitdir = os.path.abspath(".git")
2918             if not isValidGitDir(cmd.gitdir):
2919                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2920                 if os.path.exists(cmd.gitdir):
2921                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2922                     if len(cdup) > 0:
2923                         chdir(cdup);
2924
2925         if not isValidGitDir(cmd.gitdir):
2926             if isValidGitDir(cmd.gitdir + "/.git"):
2927                 cmd.gitdir += "/.git"
2928             else:
2929                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2930
2931         os.environ["GIT_DIR"] = cmd.gitdir
2932
2933     if not cmd.run(args):
2934         parser.print_help()
2935         sys.exit(2)
2936
2937
2938 if __name__ == '__main__':
2939     main()