Pileus Git - ~andy/git/blob - git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re, shutil
  14
  15 verbose = False
  16
  17 # Only labels/tags matching this will be imported/exported
  18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = ["p4"]
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += ["-u",user]
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += ["-P", password]
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += ["-p", port]
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += ["-H", host]
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += ["-c", client]
  48
  49
  50     if isinstance(cmd,basestring):
  51         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  52     else:
  53         real_cmd += cmd
  54     return real_cmd
  55
  56 def chdir(dir):
  57     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  58     # not using the shell, we have to set it ourselves.  This path could
  59     # be relative, so go there first, then figure out where we ended up.
  60     os.chdir(dir)
  61     os.environ['PWD'] = os.getcwd()
  62
  63 def die(msg):
  64     if verbose:
  65         raise Exception(msg)
  66     else:
  67         sys.stderr.write(msg + "\n")
  68         sys.exit(1)
  69
  70 def write_pipe(c, stdin):
  71     if verbose:
  72         sys.stderr.write('Writing pipe: %s\n' % str(c))
  73
  74     expand = isinstance(c,basestring)
  75     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  76     pipe = p.stdin
  77     val = pipe.write(stdin)
  78     pipe.close()
  79     if p.wait():
  80         die('Command failed: %s' % str(c))
  81
  82     return val
  83
  84 def p4_write_pipe(c, stdin):
  85     real_cmd = p4_build_cmd(c)
  86     return write_pipe(real_cmd, stdin)
  87
  88 def read_pipe(c, ignore_error=False):
  89     if verbose:
  90         sys.stderr.write('Reading pipe: %s\n' % str(c))
  91
  92     expand = isinstance(c,basestring)
  93     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  94     pipe = p.stdout
  95     val = pipe.read()
  96     if p.wait() and not ignore_error:
  97         die('Command failed: %s' % str(c))
  98
  99     return val
 100
 101 def p4_read_pipe(c, ignore_error=False):
 102     real_cmd = p4_build_cmd(c)
 103     return read_pipe(real_cmd, ignore_error)
 104
 105 def read_pipe_lines(c):
 106     if verbose:
 107         sys.stderr.write('Reading pipe: %s\n' % str(c))
 108
 109     expand = isinstance(c, basestring)
 110     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 111     pipe = p.stdout
 112     val = pipe.readlines()
 113     if pipe.close() or p.wait():
 114         die('Command failed: %s' % str(c))
 115
 116     return val
 117
 118 def p4_read_pipe_lines(c):
 119     """Specifically invoke p4 on the command supplied. """
 120     real_cmd = p4_build_cmd(c)
 121     return read_pipe_lines(real_cmd)
 122
 123 def system(cmd):
 124     expand = isinstance(cmd,basestring)
 125     if verbose:
 126         sys.stderr.write("executing %s\n" % str(cmd))
 127     subprocess.check_call(cmd, shell=expand)
 128
 129 def p4_system(cmd):
 130     """Specifically invoke p4 as the system command. """
 131     real_cmd = p4_build_cmd(cmd)
 132     expand = isinstance(real_cmd, basestring)
 133     subprocess.check_call(real_cmd, shell=expand)
 134
 135 def p4_integrate(src, dest):
 136     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 137
 138 def p4_sync(f, *options):
 139     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 140
 141 def p4_add(f):
 142     # forcibly add file names with wildcards
 143     if wildcard_present(f):
 144         p4_system(["add", "-f", f])
 145     else:
 146         p4_system(["add", f])
 147
 148 def p4_delete(f):
 149     p4_system(["delete", wildcard_encode(f)])
 150
 151 def p4_edit(f):
 152     p4_system(["edit", wildcard_encode(f)])
 153
 154 def p4_revert(f):
 155     p4_system(["revert", wildcard_encode(f)])
 156
 157 def p4_reopen(type, f):
 158     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 159
 160 #
 161 # Canonicalize the p4 type and return a tuple of the
 162 # base type, plus any modifiers.  See "p4 help filetypes"
 163 # for a list and explanation.
 164 #
 165 def split_p4_type(p4type):
 166
 167     p4_filetypes_historical = {
 168         "ctempobj": "binary+Sw",
 169         "ctext": "text+C",
 170         "cxtext": "text+Cx",
 171         "ktext": "text+k",
 172         "kxtext": "text+kx",
 173         "ltext": "text+F",
 174         "tempobj": "binary+FSw",
 175         "ubinary": "binary+F",
 176         "uresource": "resource+F",
 177         "uxbinary": "binary+Fx",
 178         "xbinary": "binary+x",
 179         "xltext": "text+Fx",
 180         "xtempobj": "binary+Swx",
 181         "xtext": "text+x",
 182         "xunicode": "unicode+x",
 183         "xutf16": "utf16+x",
 184     }
 185     if p4type in p4_filetypes_historical:
 186         p4type = p4_filetypes_historical[p4type]
 187     mods = ""
 188     s = p4type.split("+")
 189     base = s[0]
 190     mods = ""
 191     if len(s) > 1:
 192         mods = s[1]
 193     return (base, mods)
 194
 195 #
 196 # return the raw p4 type of a file (text, text+ko, etc)
 197 #
 198 def p4_type(file):
 199     results = p4CmdList(["fstat", "-T", "headType", file])
 200     return results[0]['headType']
 201
 202 #
 203 # Given a type base and modifier, return a regexp matching
 204 # the keywords that can be expanded in the file
 205 #
 206 def p4_keywords_regexp_for_type(base, type_mods):
 207     if base in ("text", "unicode", "binary"):
 208         kwords = None
 209         if "ko" in type_mods:
 210             kwords = 'Id|Header'
 211         elif "k" in type_mods:
 212             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 213         else:
 214             return None
 215         pattern = r"""
 216             \$              # Starts with a dollar, followed by...
 217             (%s)            # one of the keywords, followed by...
 218             (:[^$]+)?       # possibly an old expansion, followed by...
 219             \$              # another dollar
 220             """ % kwords
 221         return pattern
 222     else:
 223         return None
 224
 225 #
 226 # Given a file, return a regexp matching the possible
 227 # RCS keywords that will be expanded, or None for files
 228 # with kw expansion turned off.
 229 #
 230 def p4_keywords_regexp_for_file(file):
 231     if not os.path.exists(file):
 232         return None
 233     else:
 234         (type_base, type_mods) = split_p4_type(p4_type(file))
 235         return p4_keywords_regexp_for_type(type_base, type_mods)
 236
 237 def setP4ExecBit(file, mode):
 238     # Reopens an already open file and changes the execute bit to match
 239     # the execute bit setting in the passed in mode.
 240
 241     p4Type = "+x"
 242
 243     if not isModeExec(mode):
 244         p4Type = getP4OpenedType(file)
 245         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 246         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 247         if p4Type[-1] == "+":
 248             p4Type = p4Type[0:-1]
 249
 250     p4_reopen(p4Type, file)
 251
 252 def getP4OpenedType(file):
 253     # Returns the perforce file type for the given file.
 254
 255     result = p4_read_pipe(["opened", wildcard_encode(file)])
 256     match = re.match(".*\((.+)\)\r?$", result)
 257     if match:
 258         return match.group(1)
 259     else:
 260         die("Could not determine file type for %s (result: '%s')" % (file, result))
 261
 262 # Return the set of all p4 labels
 263 def getP4Labels(depotPaths):
 264     labels = set()
 265     if isinstance(depotPaths,basestring):
 266         depotPaths = [depotPaths]
 267
 268     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 269         label = l['label']
 270         labels.add(label)
 271
 272     return labels
 273
 274 # Return the set of all git tags
 275 def getGitTags():
 276     gitTags = set()
 277     for line in read_pipe_lines(["git", "tag"]):
 278         tag = line.strip()
 279         gitTags.add(tag)
 280     return gitTags
 281
 282 def diffTreePattern():
 283     # This is a simple generator for the diff tree regex pattern. This could be
 284     # a class variable if this and parseDiffTreeEntry were a part of a class.
 285     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 286     while True:
 287         yield pattern
 288
 289 def parseDiffTreeEntry(entry):
 290     """Parses a single diff tree entry into its component elements.
 291
 292     See git-diff-tree(1) manpage for details about the format of the diff
 293     output. This method returns a dictionary with the following elements:
 294
 295     src_mode - The mode of the source file
 296     dst_mode - The mode of the destination file
 297     src_sha1 - The sha1 for the source file
 298     dst_sha1 - The sha1 fr the destination file
 299     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 300     status_score - The score for the status (applicable for 'C' and 'R'
 301                    statuses). This is None if there is no score.
 302     src - The path for the source file.
 303     dst - The path for the destination file. This is only present for
 304           copy or renames. If it is not present, this is None.
 305
 306     If the pattern is not matched, None is returned."""
 307
 308     match = diffTreePattern().next().match(entry)
 309     if match:
 310         return {
 311             'src_mode': match.group(1),
 312             'dst_mode': match.group(2),
 313             'src_sha1': match.group(3),
 314             'dst_sha1': match.group(4),
 315             'status': match.group(5),
 316             'status_score': match.group(6),
 317             'src': match.group(7),
 318             'dst': match.group(10)
 319         }
 320     return None
 321
 322 def isModeExec(mode):
 323     # Returns True if the given git mode represents an executable file,
 324     # otherwise False.
 325     return mode[-3:] == "755"
 326
 327 def isModeExecChanged(src_mode, dst_mode):
 328     return isModeExec(src_mode) != isModeExec(dst_mode)
 329
 330 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 331
 332     if isinstance(cmd,basestring):
 333         cmd = "-G " + cmd
 334         expand = True
 335     else:
 336         cmd = ["-G"] + cmd
 337         expand = False
 338
 339     cmd = p4_build_cmd(cmd)
 340     if verbose:
 341         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 342
 343     # Use a temporary file to avoid deadlocks without
 344     # subprocess.communicate(), which would put another copy
 345     # of stdout into memory.
 346     stdin_file = None
 347     if stdin is not None:
 348         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 349         if isinstance(stdin,basestring):
 350             stdin_file.write(stdin)
 351         else:
 352             for i in stdin:
 353                 stdin_file.write(i + '\n')
 354         stdin_file.flush()
 355         stdin_file.seek(0)
 356
 357     p4 = subprocess.Popen(cmd,
 358                           shell=expand,
 359                           stdin=stdin_file,
 360                           stdout=subprocess.PIPE)
 361
 362     result = []
 363     try:
 364         while True:
 365             entry = marshal.load(p4.stdout)
 366             if cb is not None:
 367                 cb(entry)
 368             else:
 369                 result.append(entry)
 370     except EOFError:
 371         pass
 372     exitCode = p4.wait()
 373     if exitCode != 0:
 374         entry = {}
 375         entry["p4ExitCode"] = exitCode
 376         result.append(entry)
 377
 378     return result
 379
 380 def p4Cmd(cmd):
 381     list = p4CmdList(cmd)
 382     result = {}
 383     for entry in list:
 384         result.update(entry)
 385     return result;
 386
 387 def p4Where(depotPath):
 388     if not depotPath.endswith("/"):
 389         depotPath += "/"
 390     depotPath = depotPath + "..."
 391     outputList = p4CmdList(["where", depotPath])
 392     output = None
 393     for entry in outputList:
 394         if "depotFile" in entry:
 395             if entry["depotFile"] == depotPath:
 396                 output = entry
 397                 break
 398         elif "data" in entry:
 399             data = entry.get("data")
 400             space = data.find(" ")
 401             if data[:space] == depotPath:
 402                 output = entry
 403                 break
 404     if output == None:
 405         return ""
 406     if output["code"] == "error":
 407         return ""
 408     clientPath = ""
 409     if "path" in output:
 410         clientPath = output.get("path")
 411     elif "data" in output:
 412         data = output.get("data")
 413         lastSpace = data.rfind(" ")
 414         clientPath = data[lastSpace + 1:]
 415
 416     if clientPath.endswith("..."):
 417         clientPath = clientPath[:-3]
 418     return clientPath
 419
 420 def currentGitBranch():
 421     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 422
 423 def isValidGitDir(path):
 424     if (os.path.exists(path + "/HEAD")
 425         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 426         return True;
 427     return False
 428
 429 def parseRevision(ref):
 430     return read_pipe("git rev-parse %s" % ref).strip()
 431
 432 def branchExists(ref):
 433     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 434                      ignore_error=True)
 435     return len(rev) > 0
 436
 437 def extractLogMessageFromGitCommit(commit):
 438     logMessage = ""
 439
 440     ## fixme: title is first line of commit, not 1st paragraph.
 441     foundTitle = False
 442     for log in read_pipe_lines("git cat-file commit %s" % commit):
 443        if not foundTitle:
 444            if len(log) == 1:
 445                foundTitle = True
 446            continue
 447
 448        logMessage += log
 449     return logMessage
 450
 451 def extractSettingsGitLog(log):
 452     values = {}
 453     for line in log.split("\n"):
 454         line = line.strip()
 455         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 456         if not m:
 457             continue
 458
 459         assignments = m.group(1).split (':')
 460         for a in assignments:
 461             vals = a.split ('=')
 462             key = vals[0].strip()
 463             val = ('='.join (vals[1:])).strip()
 464             if val.endswith ('\"') and val.startswith('"'):
 465                 val = val[1:-1]
 466
 467             values[key] = val
 468
 469     paths = values.get("depot-paths")
 470     if not paths:
 471         paths = values.get("depot-path")
 472     if paths:
 473         values['depot-paths'] = paths.split(',')
 474     return values
 475
 476 def gitBranchExists(branch):
 477     proc = subprocess.Popen(["git", "rev-parse", branch],
 478                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 479     return proc.wait() == 0;
 480
 481 _gitConfig = {}
 482 def gitConfig(key, args = None): # set args to "--bool", for instance
 483     if not _gitConfig.has_key(key):
 484         argsFilter = ""
 485         if args != None:
 486             argsFilter = "%s " % args
 487         cmd = "git config %s%s" % (argsFilter, key)
 488         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 489     return _gitConfig[key]
 490
 491 def gitConfigList(key):
 492     if not _gitConfig.has_key(key):
 493         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 494     return _gitConfig[key]
 495
 496 def p4BranchesInGit(branchesAreInRemotes = True):
 497     branches = {}
 498
 499     cmdline = "git rev-parse --symbolic "
 500     if branchesAreInRemotes:
 501         cmdline += " --remotes"
 502     else:
 503         cmdline += " --branches"
 504
 505     for line in read_pipe_lines(cmdline):
 506         line = line.strip()
 507
 508         ## only import to p4/
 509         if not line.startswith('p4/') or line == "p4/HEAD":
 510             continue
 511         branch = line
 512
 513         # strip off p4
 514         branch = re.sub ("^p4/", "", line)
 515
 516         branches[branch] = parseRevision(line)
 517     return branches
 518
 519 def findUpstreamBranchPoint(head = "HEAD"):
 520     branches = p4BranchesInGit()
 521     # map from depot-path to branch name
 522     branchByDepotPath = {}
 523     for branch in branches.keys():
 524         tip = branches[branch]
 525         log = extractLogMessageFromGitCommit(tip)
 526         settings = extractSettingsGitLog(log)
 527         if settings.has_key("depot-paths"):
 528             paths = ",".join(settings["depot-paths"])
 529             branchByDepotPath[paths] = "remotes/p4/" + branch
 530
 531     settings = None
 532     parent = 0
 533     while parent < 65535:
 534         commit = head + "~%s" % parent
 535         log = extractLogMessageFromGitCommit(commit)
 536         settings = extractSettingsGitLog(log)
 537         if settings.has_key("depot-paths"):
 538             paths = ",".join(settings["depot-paths"])
 539             if branchByDepotPath.has_key(paths):
 540                 return [branchByDepotPath[paths], settings]
 541
 542         parent = parent + 1
 543
 544     return ["", settings]
 545
 546 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 547     if not silent:
 548         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 549                % localRefPrefix)
 550
 551     originPrefix = "origin/p4/"
 552
 553     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 554         line = line.strip()
 555         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 556             continue
 557
 558         headName = line[len(originPrefix):]
 559         remoteHead = localRefPrefix + headName
 560         originHead = line
 561
 562         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 563         if (not original.has_key('depot-paths')
 564             or not original.has_key('change')):
 565             continue
 566
 567         update = False
 568         if not gitBranchExists(remoteHead):
 569             if verbose:
 570                 print "creating %s" % remoteHead
 571             update = True
 572         else:
 573             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 574             if settings.has_key('change') > 0:
 575                 if settings['depot-paths'] == original['depot-paths']:
 576                     originP4Change = int(original['change'])
 577                     p4Change = int(settings['change'])
 578                     if originP4Change > p4Change:
 579                         print ("%s (%s) is newer than %s (%s). "
 580                                "Updating p4 branch from origin."
 581                                % (originHead, originP4Change,
 582                                   remoteHead, p4Change))
 583                         update = True
 584                 else:
 585                     print ("Ignoring: %s was imported from %s while "
 586                            "%s was imported from %s"
 587                            % (originHead, ','.join(original['depot-paths']),
 588                               remoteHead, ','.join(settings['depot-paths'])))
 589
 590         if update:
 591             system("git update-ref %s %s" % (remoteHead, originHead))
 592
 593 def originP4BranchesExist():
 594         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 595
 596 def p4ChangesForPaths(depotPaths, changeRange):
 597     assert depotPaths
 598     cmd = ['changes']
 599     for p in depotPaths:
 600         cmd += ["%s...%s" % (p, changeRange)]
 601     output = p4_read_pipe_lines(cmd)
 602
 603     changes = {}
 604     for line in output:
 605         changeNum = int(line.split(" ")[1])
 606         changes[changeNum] = True
 607
 608     changelist = changes.keys()
 609     changelist.sort()
 610     return changelist
 611
 612 def p4PathStartsWith(path, prefix):
 613     # This method tries to remedy a potential mixed-case issue:
 614     #
 615     # If UserA adds  //depot/DirA/file1
 616     # and UserB adds //depot/dira/file2
 617     #
 618     # we may or may not have a problem. If you have core.ignorecase=true,
 619     # we treat DirA and dira as the same directory
 620     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 621     if ignorecase:
 622         return path.lower().startswith(prefix.lower())
 623     return path.startswith(prefix)
 624
 625 def getClientSpec():
 626     """Look at the p4 client spec, create a View() object that contains
 627        all the mappings, and return it."""
 628
 629     specList = p4CmdList("client -o")
 630     if len(specList) != 1:
 631         die('Output from "client -o" is %d lines, expecting 1' %
 632             len(specList))
 633
 634     # dictionary of all client parameters
 635     entry = specList[0]
 636
 637     # just the keys that start with "View"
 638     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 639
 640     # hold this new View
 641     view = View()
 642
 643     # append the lines, in order, to the view
 644     for view_num in range(len(view_keys)):
 645         k = "View%d" % view_num
 646         if k not in view_keys:
 647             die("Expected view key %s missing" % k)
 648         view.append(entry[k])
 649
 650     return view
 651
 652 def getClientRoot():
 653     """Grab the client directory."""
 654
 655     output = p4CmdList("client -o")
 656     if len(output) != 1:
 657         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 658
 659     entry = output[0]
 660     if "Root" not in entry:
 661         die('Client has no "Root"')
 662
 663     return entry["Root"]
 664
 665 #
 666 # P4 wildcards are not allowed in filenames.  P4 complains
 667 # if you simply add them, but you can force it with "-f", in
 668 # which case it translates them into %xx encoding internally.
 669 #
 670 def wildcard_decode(path):
 671     # Search for and fix just these four characters.  Do % last so
 672     # that fixing it does not inadvertently create new %-escapes.
 673     # Cannot have * in a filename in windows; untested as to
 674     # what p4 would do in such a case.
 675     if not platform.system() == "Windows":
 676         path = path.replace("%2A", "*")
 677     path = path.replace("%23", "#") \
 678                .replace("%40", "@") \
 679                .replace("%25", "%")
 680     return path
 681
 682 def wildcard_encode(path):
 683     # do % first to avoid double-encoding the %s introduced here
 684     path = path.replace("%", "%25") \
 685                .replace("*", "%2A") \
 686                .replace("#", "%23") \
 687                .replace("@", "%40")
 688     return path
 689
 690 def wildcard_present(path):
 691     return path.translate(None, "*#@%") != path
 692
 693 class Command:
 694     def __init__(self):
 695         self.usage = "usage: %prog [options]"
 696         self.needsGit = True
 697         self.verbose = False
 698
 699 class P4UserMap:
 700     def __init__(self):
 701         self.userMapFromPerforceServer = False
 702         self.myP4UserId = None
 703
 704     def p4UserId(self):
 705         if self.myP4UserId:
 706             return self.myP4UserId
 707
 708         results = p4CmdList("user -o")
 709         for r in results:
 710             if r.has_key('User'):
 711                 self.myP4UserId = r['User']
 712                 return r['User']
 713         die("Could not find your p4 user id")
 714
 715     def p4UserIsMe(self, p4User):
 716         # return True if the given p4 user is actually me
 717         me = self.p4UserId()
 718         if not p4User or p4User != me:
 719             return False
 720         else:
 721             return True
 722
 723     def getUserCacheFilename(self):
 724         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 725         return home + "/.gitp4-usercache.txt"
 726
 727     def getUserMapFromPerforceServer(self):
 728         if self.userMapFromPerforceServer:
 729             return
 730         self.users = {}
 731         self.emails = {}
 732
 733         for output in p4CmdList("users"):
 734             if not output.has_key("User"):
 735                 continue
 736             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 737             self.emails[output["Email"]] = output["User"]
 738
 739
 740         s = ''
 741         for (key, val) in self.users.items():
 742             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 743
 744         open(self.getUserCacheFilename(), "wb").write(s)
 745         self.userMapFromPerforceServer = True
 746
 747     def loadUserMapFromCache(self):
 748         self.users = {}
 749         self.userMapFromPerforceServer = False
 750         try:
 751             cache = open(self.getUserCacheFilename(), "rb")
 752             lines = cache.readlines()
 753             cache.close()
 754             for line in lines:
 755                 entry = line.strip().split("\t")
 756                 self.users[entry[0]] = entry[1]
 757         except IOError:
 758             self.getUserMapFromPerforceServer()
 759
 760 class P4Debug(Command):
 761     def __init__(self):
 762         Command.__init__(self)
 763         self.options = []
 764         self.description = "A tool to debug the output of p4 -G."
 765         self.needsGit = False
 766
 767     def run(self, args):
 768         j = 0
 769         for output in p4CmdList(args):
 770             print 'Element: %d' % j
 771             j += 1
 772             print output
 773         return True
 774
 775 class P4RollBack(Command):
 776     def __init__(self):
 777         Command.__init__(self)
 778         self.options = [
 779             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 780         ]
 781         self.description = "A tool to debug the multi-branch import. Don't use :)"
 782         self.rollbackLocalBranches = False
 783
 784     def run(self, args):
 785         if len(args) != 1:
 786             return False
 787         maxChange = int(args[0])
 788
 789         if "p4ExitCode" in p4Cmd("changes -m 1"):
 790             die("Problems executing p4");
 791
 792         if self.rollbackLocalBranches:
 793             refPrefix = "refs/heads/"
 794             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 795         else:
 796             refPrefix = "refs/remotes/"
 797             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 798
 799         for line in lines:
 800             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 801                 line = line.strip()
 802                 ref = refPrefix + line
 803                 log = extractLogMessageFromGitCommit(ref)
 804                 settings = extractSettingsGitLog(log)
 805
 806                 depotPaths = settings['depot-paths']
 807                 change = settings['change']
 808
 809                 changed = False
 810
 811                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 812                                                            for p in depotPaths]))) == 0:
 813                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 814                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 815                     continue
 816
 817                 while change and int(change) > maxChange:
 818                     changed = True
 819                     if self.verbose:
 820                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 821                     system("git update-ref %s \"%s^\"" % (ref, ref))
 822                     log = extractLogMessageFromGitCommit(ref)
 823                     settings =  extractSettingsGitLog(log)
 824
 825
 826                     depotPaths = settings['depot-paths']
 827                     change = settings['change']
 828
 829                 if changed:
 830                     print "%s rewound to %s" % (ref, change)
 831
 832         return True
 833
 834 class P4Submit(Command, P4UserMap):
 835     def __init__(self):
 836         Command.__init__(self)
 837         P4UserMap.__init__(self)
 838         self.options = [
 839                 optparse.make_option("--origin", dest="origin"),
 840                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 841                 # preserve the user, requires relevant p4 permissions
 842                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 843                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 844         ]
 845         self.description = "Submit changes from git to the perforce depot."
 846         self.usage += " [name of git branch to submit into perforce depot]"
 847         self.interactive = True
 848         self.origin = ""
 849         self.detectRenames = False
 850         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 851         self.isWindows = (platform.system() == "Windows")
 852         self.exportLabels = False
 853
 854     def check(self):
 855         if len(p4CmdList("opened ...")) > 0:
 856             die("You have files opened with perforce! Close them before starting the sync.")
 857
 858     # replaces everything between 'Description:' and the next P4 submit template field with the
 859     # commit message
 860     def prepareLogMessage(self, template, message):
 861         result = ""
 862
 863         inDescriptionSection = False
 864
 865         for line in template.split("\n"):
 866             if line.startswith("#"):
 867                 result += line + "\n"
 868                 continue
 869
 870             if inDescriptionSection:
 871                 if line.startswith("Files:") or line.startswith("Jobs:"):
 872                     inDescriptionSection = False
 873                 else:
 874                     continue
 875             else:
 876                 if line.startswith("Description:"):
 877                     inDescriptionSection = True
 878                     line += "\n"
 879                     for messageLine in message.split("\n"):
 880                         line += "\t" + messageLine + "\n"
 881
 882             result += line + "\n"
 883
 884         return result
 885
 886     def patchRCSKeywords(self, file, pattern):
 887         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
 888         (handle, outFileName) = tempfile.mkstemp(dir='.')
 889         try:
 890             outFile = os.fdopen(handle, "w+")
 891             inFile = open(file, "r")
 892             regexp = re.compile(pattern, re.VERBOSE)
 893             for line in inFile.readlines():
 894                 line = regexp.sub(r'$\1$', line)
 895                 outFile.write(line)
 896             inFile.close()
 897             outFile.close()
 898             # Forcibly overwrite the original file
 899             os.unlink(file)
 900             shutil.move(outFileName, file)
 901         except:
 902             # cleanup our temporary file
 903             os.unlink(outFileName)
 904             print "Failed to strip RCS keywords in %s" % file
 905             raise
 906
 907         print "Patched up RCS keywords in %s" % file
 908
 909     def p4UserForCommit(self,id):
 910         # Return the tuple (perforce user,git email) for a given git commit id
 911         self.getUserMapFromPerforceServer()
 912         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 913         gitEmail = gitEmail.strip()
 914         if not self.emails.has_key(gitEmail):
 915             return (None,gitEmail)
 916         else:
 917             return (self.emails[gitEmail],gitEmail)
 918
 919     def checkValidP4Users(self,commits):
 920         # check if any git authors cannot be mapped to p4 users
 921         for id in commits:
 922             (user,email) = self.p4UserForCommit(id)
 923             if not user:
 924                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 925                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 926                     print "%s" % msg
 927                 else:
 928                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 929
 930     def lastP4Changelist(self):
 931         # Get back the last changelist number submitted in this client spec. This
 932         # then gets used to patch up the username in the change. If the same
 933         # client spec is being used by multiple processes then this might go
 934         # wrong.
 935         results = p4CmdList("client -o")        # find the current client
 936         client = None
 937         for r in results:
 938             if r.has_key('Client'):
 939                 client = r['Client']
 940                 break
 941         if not client:
 942             die("could not get client spec")
 943         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 944         for r in results:
 945             if r.has_key('change'):
 946                 return r['change']
 947         die("Could not get changelist number for last submit - cannot patch up user details")
 948
 949     def modifyChangelistUser(self, changelist, newUser):
 950         # fixup the user field of a changelist after it has been submitted.
 951         changes = p4CmdList("change -o %s" % changelist)
 952         if len(changes) != 1:
 953             die("Bad output from p4 change modifying %s to user %s" %
 954                 (changelist, newUser))
 955
 956         c = changes[0]
 957         if c['User'] == newUser: return   # nothing to do
 958         c['User'] = newUser
 959         input = marshal.dumps(c)
 960
 961         result = p4CmdList("change -f -i", stdin=input)
 962         for r in result:
 963             if r.has_key('code'):
 964                 if r['code'] == 'error':
 965                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 966             if r.has_key('data'):
 967                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 968                 return
 969         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 970
 971     def canChangeChangelists(self):
 972         # check to see if we have p4 admin or super-user permissions, either of
 973         # which are required to modify changelists.
 974         results = p4CmdList(["protects", self.depotPath])
 975         for r in results:
 976             if r.has_key('perm'):
 977                 if r['perm'] == 'admin':
 978                     return 1
 979                 if r['perm'] == 'super':
 980                     return 1
 981         return 0
 982
 983     def prepareSubmitTemplate(self):
 984         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 985         template = ""
 986         inFilesSection = False
 987         for line in p4_read_pipe_lines(['change', '-o']):
 988             if line.endswith("\r\n"):
 989                 line = line[:-2] + "\n"
 990             if inFilesSection:
 991                 if line.startswith("\t"):
 992                     # path starts and ends with a tab
 993                     path = line[1:]
 994                     lastTab = path.rfind("\t")
 995                     if lastTab != -1:
 996                         path = path[:lastTab]
 997                         if not p4PathStartsWith(path, self.depotPath):
 998                             continue
 999                 else:
1000                     inFilesSection = False
1001             else:
1002                 if line.startswith("Files:"):
1003                     inFilesSection = True
1004
1005             template += line
1006
1007         return template
1008
1009     def edit_template(self, template_file):
1010         """Invoke the editor to let the user change the submission
1011            message.  Return true if okay to continue with the submit."""
1012
1013         # if configured to skip the editing part, just submit
1014         if gitConfig("git-p4.skipSubmitEdit") == "true":
1015             return True
1016
1017         # look at the modification time, to check later if the user saved
1018         # the file
1019         mtime = os.stat(template_file).st_mtime
1020
1021         # invoke the editor
1022         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1023             editor = os.environ.get("P4EDITOR")
1024         else:
1025             editor = read_pipe("git var GIT_EDITOR").strip()
1026         system(editor + " " + template_file)
1027
1028         # If the file was not saved, prompt to see if this patch should
1029         # be skipped.  But skip this verification step if configured so.
1030         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1031             return True
1032
1033         # modification time updated means user saved the file
1034         if os.stat(template_file).st_mtime > mtime:
1035             return True
1036
1037         while True:
1038             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1039             if response == 'y':
1040                 return True
1041             if response == 'n':
1042                 return False
1043
1044     def applyCommit(self, id):
1045         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1046
1047         (p4User, gitEmail) = self.p4UserForCommit(id)
1048
1049
1050         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1051         filesToAdd = set()
1052         filesToDelete = set()
1053         editedFiles = set()
1054         pureRenameCopy = set()
1055         filesToChangeExecBit = {}
1056
1057         for line in diff:
1058             diff = parseDiffTreeEntry(line)
1059             modifier = diff['status']
1060             path = diff['src']
1061             if modifier == "M":
1062                 p4_edit(path)
1063                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1064                     filesToChangeExecBit[path] = diff['dst_mode']
1065                 editedFiles.add(path)
1066             elif modifier == "A":
1067                 filesToAdd.add(path)
1068                 filesToChangeExecBit[path] = diff['dst_mode']
1069                 if path in filesToDelete:
1070                     filesToDelete.remove(path)
1071             elif modifier == "D":
1072                 filesToDelete.add(path)
1073                 if path in filesToAdd:
1074                     filesToAdd.remove(path)
1075             elif modifier == "C":
1076                 src, dest = diff['src'], diff['dst']
1077                 p4_integrate(src, dest)
1078                 pureRenameCopy.add(dest)
1079                 if diff['src_sha1'] != diff['dst_sha1']:
1080                     p4_edit(dest)
1081                     pureRenameCopy.discard(dest)
1082                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1083                     p4_edit(dest)
1084                     pureRenameCopy.discard(dest)
1085                     filesToChangeExecBit[dest] = diff['dst_mode']
1086                 os.unlink(dest)
1087                 editedFiles.add(dest)
1088             elif modifier == "R":
1089                 src, dest = diff['src'], diff['dst']
1090                 p4_integrate(src, dest)
1091                 if diff['src_sha1'] != diff['dst_sha1']:
1092                     p4_edit(dest)
1093                 else:
1094                     pureRenameCopy.add(dest)
1095                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1096                     p4_edit(dest)
1097                     filesToChangeExecBit[dest] = diff['dst_mode']
1098                 os.unlink(dest)
1099                 editedFiles.add(dest)
1100                 filesToDelete.add(src)
1101             else:
1102                 die("unknown modifier %s for %s" % (modifier, path))
1103
1104         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1105         patchcmd = diffcmd + " | git apply "
1106         tryPatchCmd = patchcmd + "--check -"
1107         applyPatchCmd = patchcmd + "--check --apply -"
1108         patch_succeeded = True
1109
1110         if os.system(tryPatchCmd) != 0:
1111             fixed_rcs_keywords = False
1112             patch_succeeded = False
1113             print "Unfortunately applying the change failed!"
1114
1115             # Patch failed, maybe it's just RCS keyword woes. Look through
1116             # the patch to see if that's possible.
1117             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1118                 file = None
1119                 pattern = None
1120                 kwfiles = {}
1121                 for file in editedFiles | filesToDelete:
1122                     # did this file's delta contain RCS keywords?
1123                     pattern = p4_keywords_regexp_for_file(file)
1124
1125                     if pattern:
1126                         # this file is a possibility...look for RCS keywords.
1127                         regexp = re.compile(pattern, re.VERBOSE)
1128                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1129                             if regexp.search(line):
1130                                 if verbose:
1131                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1132                                 kwfiles[file] = pattern
1133                                 break
1134
1135                 for file in kwfiles:
1136                     if verbose:
1137                         print "zapping %s with %s" % (line,pattern)
1138                     self.patchRCSKeywords(file, kwfiles[file])
1139                     fixed_rcs_keywords = True
1140
1141             if fixed_rcs_keywords:
1142                 print "Retrying the patch with RCS keywords cleaned up"
1143                 if os.system(tryPatchCmd) == 0:
1144                     patch_succeeded = True
1145
1146         if not patch_succeeded:
1147             print "What do you want to do?"
1148             response = "x"
1149             while response != "s" and response != "a" and response != "w":
1150                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1151                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1152             if response == "s":
1153                 print "Skipping! Good luck with the next patches..."
1154                 for f in editedFiles:
1155                     p4_revert(f)
1156                 for f in filesToAdd:
1157                     os.remove(f)
1158                 return
1159             elif response == "a":
1160                 os.system(applyPatchCmd)
1161                 if len(filesToAdd) > 0:
1162                     print "You may also want to call p4 add on the following files:"
1163                     print " ".join(filesToAdd)
1164                 if len(filesToDelete):
1165                     print "The following files should be scheduled for deletion with p4 delete:"
1166                     print " ".join(filesToDelete)
1167                 die("Please resolve and submit the conflict manually and "
1168                     + "continue afterwards with git p4 submit --continue")
1169             elif response == "w":
1170                 system(diffcmd + " > patch.txt")
1171                 print "Patch saved to patch.txt in %s !" % self.clientPath
1172                 die("Please resolve and submit the conflict manually and "
1173                     "continue afterwards with git p4 submit --continue")
1174
1175         system(applyPatchCmd)
1176
1177         for f in filesToAdd:
1178             p4_add(f)
1179         for f in filesToDelete:
1180             p4_revert(f)
1181             p4_delete(f)
1182
1183         # Set/clear executable bits
1184         for f in filesToChangeExecBit.keys():
1185             mode = filesToChangeExecBit[f]
1186             setP4ExecBit(f, mode)
1187
1188         logMessage = extractLogMessageFromGitCommit(id)
1189         logMessage = logMessage.strip()
1190
1191         template = self.prepareSubmitTemplate()
1192
1193         if self.interactive:
1194             submitTemplate = self.prepareLogMessage(template, logMessage)
1195
1196             if self.preserveUser:
1197                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1198
1199             if os.environ.has_key("P4DIFF"):
1200                 del(os.environ["P4DIFF"])
1201             diff = ""
1202             for editedFile in editedFiles:
1203                 diff += p4_read_pipe(['diff', '-du',
1204                                       wildcard_encode(editedFile)])
1205
1206             newdiff = ""
1207             for newFile in filesToAdd:
1208                 newdiff += "==== new file ====\n"
1209                 newdiff += "--- /dev/null\n"
1210                 newdiff += "+++ %s\n" % newFile
1211                 f = open(newFile, "r")
1212                 for line in f.readlines():
1213                     newdiff += "+" + line
1214                 f.close()
1215
1216             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1217                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1218                 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1219                 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1220
1221             separatorLine = "######## everything below this line is just the diff #######\n"
1222
1223             (handle, fileName) = tempfile.mkstemp()
1224             tmpFile = os.fdopen(handle, "w+")
1225             if self.isWindows:
1226                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1227                 separatorLine = separatorLine.replace("\n", "\r\n")
1228                 newdiff = newdiff.replace("\n", "\r\n")
1229             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1230             tmpFile.close()
1231
1232             if self.edit_template(fileName):
1233                 # read the edited message and submit
1234                 tmpFile = open(fileName, "rb")
1235                 message = tmpFile.read()
1236                 tmpFile.close()
1237                 submitTemplate = message[:message.index(separatorLine)]
1238                 if self.isWindows:
1239                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1240                 p4_write_pipe(['submit', '-i'], submitTemplate)
1241
1242                 if self.preserveUser:
1243                     if p4User:
1244                         # Get last changelist number. Cannot easily get it from
1245                         # the submit command output as the output is
1246                         # unmarshalled.
1247                         changelist = self.lastP4Changelist()
1248                         self.modifyChangelistUser(changelist, p4User)
1249
1250                 # The rename/copy happened by applying a patch that created a
1251                 # new file.  This leaves it writable, which confuses p4.
1252                 for f in pureRenameCopy:
1253                     p4_sync(f, "-f")
1254
1255             else:
1256                 # skip this patch
1257                 print "Submission cancelled, undoing p4 changes."
1258                 for f in editedFiles:
1259                     p4_revert(f)
1260                 for f in filesToAdd:
1261                     p4_revert(f)
1262                     os.remove(f)
1263
1264             os.remove(fileName)
1265         else:
1266             fileName = "submit.txt"
1267             file = open(fileName, "w+")
1268             file.write(self.prepareLogMessage(template, logMessage))
1269             file.close()
1270             print ("Perforce submit template written as %s. "
1271                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1272                    % (fileName, fileName))
1273
1274     # Export git tags as p4 labels. Create a p4 label and then tag
1275     # with that.
1276     def exportGitTags(self, gitTags):
1277         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1278         if len(validLabelRegexp) == 0:
1279             validLabelRegexp = defaultLabelRegexp
1280         m = re.compile(validLabelRegexp)
1281
1282         for name in gitTags:
1283
1284             if not m.match(name):
1285                 if verbose:
1286                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1287                 continue
1288
1289             # Get the p4 commit this corresponds to
1290             logMessage = extractLogMessageFromGitCommit(name)
1291             values = extractSettingsGitLog(logMessage)
1292
1293             if not values.has_key('change'):
1294                 # a tag pointing to something not sent to p4; ignore
1295                 if verbose:
1296                     print "git tag %s does not give a p4 commit" % name
1297                 continue
1298             else:
1299                 changelist = values['change']
1300
1301             # Get the tag details.
1302             inHeader = True
1303             isAnnotated = False
1304             body = []
1305             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1306                 l = l.strip()
1307                 if inHeader:
1308                     if re.match(r'tag\s+', l):
1309                         isAnnotated = True
1310                     elif re.match(r'\s*$', l):
1311                         inHeader = False
1312                         continue
1313                 else:
1314                     body.append(l)
1315
1316             if not isAnnotated:
1317                 body = ["lightweight tag imported by git p4\n"]
1318
1319             # Create the label - use the same view as the client spec we are using
1320             clientSpec = getClientSpec()
1321
1322             labelTemplate  = "Label: %s\n" % name
1323             labelTemplate += "Description:\n"
1324             for b in body:
1325                 labelTemplate += "\t" + b + "\n"
1326             labelTemplate += "View:\n"
1327             for mapping in clientSpec.mappings:
1328                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1329
1330             p4_write_pipe(["label", "-i"], labelTemplate)
1331
1332             # Use the label
1333             p4_system(["tag", "-l", name] +
1334                       ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1335
1336             if verbose:
1337                 print "created p4 label for tag %s" % name
1338
1339     def run(self, args):
1340         if len(args) == 0:
1341             self.master = currentGitBranch()
1342             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1343                 die("Detecting current git branch failed!")
1344         elif len(args) == 1:
1345             self.master = args[0]
1346             if not branchExists(self.master):
1347                 die("Branch %s does not exist" % self.master)
1348         else:
1349             return False
1350
1351         allowSubmit = gitConfig("git-p4.allowSubmit")
1352         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1353             die("%s is not in git-p4.allowSubmit" % self.master)
1354
1355         [upstream, settings] = findUpstreamBranchPoint()
1356         self.depotPath = settings['depot-paths'][0]
1357         if len(self.origin) == 0:
1358             self.origin = upstream
1359
1360         if self.preserveUser:
1361             if not self.canChangeChangelists():
1362                 die("Cannot preserve user names without p4 super-user or admin permissions")
1363
1364         if self.verbose:
1365             print "Origin branch is " + self.origin
1366
1367         if len(self.depotPath) == 0:
1368             print "Internal error: cannot locate perforce depot path from existing branches"
1369             sys.exit(128)
1370
1371         self.useClientSpec = False
1372         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1373             self.useClientSpec = True
1374         if self.useClientSpec:
1375             self.clientSpecDirs = getClientSpec()
1376
1377         if self.useClientSpec:
1378             # all files are relative to the client spec
1379             self.clientPath = getClientRoot()
1380         else:
1381             self.clientPath = p4Where(self.depotPath)
1382
1383         if self.clientPath == "":
1384             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1385
1386         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1387         self.oldWorkingDirectory = os.getcwd()
1388
1389         # ensure the clientPath exists
1390         new_client_dir = False
1391         if not os.path.exists(self.clientPath):
1392             new_client_dir = True
1393             os.makedirs(self.clientPath)
1394
1395         chdir(self.clientPath)
1396         print "Synchronizing p4 checkout..."
1397         if new_client_dir:
1398             # old one was destroyed, and maybe nobody told p4
1399             p4_sync("...", "-f")
1400         else:
1401             p4_sync("...")
1402         self.check()
1403
1404         commits = []
1405         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1406             commits.append(line.strip())
1407         commits.reverse()
1408
1409         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1410             self.checkAuthorship = False
1411         else:
1412             self.checkAuthorship = True
1413
1414         if self.preserveUser:
1415             self.checkValidP4Users(commits)
1416
1417         #
1418         # Build up a set of options to be passed to diff when
1419         # submitting each commit to p4.
1420         #
1421         if self.detectRenames:
1422             # command-line -M arg
1423             self.diffOpts = "-M"
1424         else:
1425             # If not explicitly set check the config variable
1426             detectRenames = gitConfig("git-p4.detectRenames")
1427
1428             if detectRenames.lower() == "false" or detectRenames == "":
1429                 self.diffOpts = ""
1430             elif detectRenames.lower() == "true":
1431                 self.diffOpts = "-M"
1432             else:
1433                 self.diffOpts = "-M%s" % detectRenames
1434
1435         # no command-line arg for -C or --find-copies-harder, just
1436         # config variables
1437         detectCopies = gitConfig("git-p4.detectCopies")
1438         if detectCopies.lower() == "false" or detectCopies == "":
1439             pass
1440         elif detectCopies.lower() == "true":
1441             self.diffOpts += " -C"
1442         else:
1443             self.diffOpts += " -C%s" % detectCopies
1444
1445         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1446             self.diffOpts += " --find-copies-harder"
1447
1448         while len(commits) > 0:
1449             commit = commits[0]
1450             commits = commits[1:]
1451             self.applyCommit(commit)
1452             if not self.interactive:
1453                 break
1454
1455         if len(commits) == 0:
1456             print "All changes applied!"
1457             chdir(self.oldWorkingDirectory)
1458
1459             sync = P4Sync()
1460             sync.run([])
1461
1462             rebase = P4Rebase()
1463             rebase.rebase()
1464
1465         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1466             self.exportLabels = True
1467
1468         if self.exportLabels:
1469             p4Labels = getP4Labels(self.depotPath)
1470             gitTags = getGitTags()
1471
1472             missingGitTags = gitTags - p4Labels
1473             self.exportGitTags(missingGitTags)
1474
1475         return True
1476
1477 class View(object):
1478     """Represent a p4 view ("p4 help views"), and map files in a
1479        repo according to the view."""
1480
1481     class Path(object):
1482         """A depot or client path, possibly containing wildcards.
1483            The only one supported is ... at the end, currently.
1484            Initialize with the full path, with //depot or //client."""
1485
1486         def __init__(self, path, is_depot):
1487             self.path = path
1488             self.is_depot = is_depot
1489             self.find_wildcards()
1490             # remember the prefix bit, useful for relative mappings
1491             m = re.match("(//[^/]+/)", self.path)
1492             if not m:
1493                 die("Path %s does not start with //prefix/" % self.path)
1494             prefix = m.group(1)
1495             if not self.is_depot:
1496                 # strip //client/ on client paths
1497                 self.path = self.path[len(prefix):]
1498
1499         def find_wildcards(self):
1500             """Make sure wildcards are valid, and set up internal
1501                variables."""
1502
1503             self.ends_triple_dot = False
1504             # There are three wildcards allowed in p4 views
1505             # (see "p4 help views").  This code knows how to
1506             # handle "..." (only at the end), but cannot deal with
1507             # "%%n" or "*".  Only check the depot_side, as p4 should
1508             # validate that the client_side matches too.
1509             if re.search(r'%%[1-9]', self.path):
1510                 die("Can't handle %%n wildcards in view: %s" % self.path)
1511             if self.path.find("*") >= 0:
1512                 die("Can't handle * wildcards in view: %s" % self.path)
1513             triple_dot_index = self.path.find("...")
1514             if triple_dot_index >= 0:
1515                 if triple_dot_index != len(self.path) - 3:
1516                     die("Can handle only single ... wildcard, at end: %s" %
1517                         self.path)
1518                 self.ends_triple_dot = True
1519
1520         def ensure_compatible(self, other_path):
1521             """Make sure the wildcards agree."""
1522             if self.ends_triple_dot != other_path.ends_triple_dot:
1523                  die("Both paths must end with ... if either does;\n" +
1524                      "paths: %s %s" % (self.path, other_path.path))
1525
1526         def match_wildcards(self, test_path):
1527             """See if this test_path matches us, and fill in the value
1528                of the wildcards if so.  Returns a tuple of
1529                (True|False, wildcards[]).  For now, only the ... at end
1530                is supported, so at most one wildcard."""
1531             if self.ends_triple_dot:
1532                 dotless = self.path[:-3]
1533                 if test_path.startswith(dotless):
1534                     wildcard = test_path[len(dotless):]
1535                     return (True, [ wildcard ])
1536             else:
1537                 if test_path == self.path:
1538                     return (True, [])
1539             return (False, [])
1540
1541         def match(self, test_path):
1542             """Just return if it matches; don't bother with the wildcards."""
1543             b, _ = self.match_wildcards(test_path)
1544             return b
1545
1546         def fill_in_wildcards(self, wildcards):
1547             """Return the relative path, with the wildcards filled in
1548                if there are any."""
1549             if self.ends_triple_dot:
1550                 return self.path[:-3] + wildcards[0]
1551             else:
1552                 return self.path
1553
1554     class Mapping(object):
1555         def __init__(self, depot_side, client_side, overlay, exclude):
1556             # depot_side is without the trailing /... if it had one
1557             self.depot_side = View.Path(depot_side, is_depot=True)
1558             self.client_side = View.Path(client_side, is_depot=False)
1559             self.overlay = overlay  # started with "+"
1560             self.exclude = exclude  # started with "-"
1561             assert not (self.overlay and self.exclude)
1562             self.depot_side.ensure_compatible(self.client_side)
1563
1564         def __str__(self):
1565             c = " "
1566             if self.overlay:
1567                 c = "+"
1568             if self.exclude:
1569                 c = "-"
1570             return "View.Mapping: %s%s -> %s" % \
1571                    (c, self.depot_side.path, self.client_side.path)
1572
1573         def map_depot_to_client(self, depot_path):
1574             """Calculate the client path if using this mapping on the
1575                given depot path; does not consider the effect of other
1576                mappings in a view.  Even excluded mappings are returned."""
1577             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1578             if not matches:
1579                 return ""
1580             client_path = self.client_side.fill_in_wildcards(wildcards)
1581             return client_path
1582
1583     #
1584     # View methods
1585     #
1586     def __init__(self):
1587         self.mappings = []
1588
1589     def append(self, view_line):
1590         """Parse a view line, splitting it into depot and client
1591            sides.  Append to self.mappings, preserving order."""
1592
1593         # Split the view line into exactly two words.  P4 enforces
1594         # structure on these lines that simplifies this quite a bit.
1595         #
1596         # Either or both words may be double-quoted.
1597         # Single quotes do not matter.
1598         # Double-quote marks cannot occur inside the words.
1599         # A + or - prefix is also inside the quotes.
1600         # There are no quotes unless they contain a space.
1601         # The line is already white-space stripped.
1602         # The two words are separated by a single space.
1603         #
1604         if view_line[0] == '"':
1605             # First word is double quoted.  Find its end.
1606             close_quote_index = view_line.find('"', 1)
1607             if close_quote_index <= 0:
1608                 die("No first-word closing quote found: %s" % view_line)
1609             depot_side = view_line[1:close_quote_index]
1610             # skip closing quote and space
1611             rhs_index = close_quote_index + 1 + 1
1612         else:
1613             space_index = view_line.find(" ")
1614             if space_index <= 0:
1615                 die("No word-splitting space found: %s" % view_line)
1616             depot_side = view_line[0:space_index]
1617             rhs_index = space_index + 1
1618
1619         if view_line[rhs_index] == '"':
1620             # Second word is double quoted.  Make sure there is a
1621             # double quote at the end too.
1622             if not view_line.endswith('"'):
1623                 die("View line with rhs quote should end with one: %s" %
1624                     view_line)
1625             # skip the quotes
1626             client_side = view_line[rhs_index+1:-1]
1627         else:
1628             client_side = view_line[rhs_index:]
1629
1630         # prefix + means overlay on previous mapping
1631         overlay = False
1632         if depot_side.startswith("+"):
1633             overlay = True
1634             depot_side = depot_side[1:]
1635
1636         # prefix - means exclude this path
1637         exclude = False
1638         if depot_side.startswith("-"):
1639             exclude = True
1640             depot_side = depot_side[1:]
1641
1642         m = View.Mapping(depot_side, client_side, overlay, exclude)
1643         self.mappings.append(m)
1644
1645     def map_in_client(self, depot_path):
1646         """Return the relative location in the client where this
1647            depot file should live.  Returns "" if the file should
1648            not be mapped in the client."""
1649
1650         paths_filled = []
1651         client_path = ""
1652
1653         # look at later entries first
1654         for m in self.mappings[::-1]:
1655
1656             # see where will this path end up in the client
1657             p = m.map_depot_to_client(depot_path)
1658
1659             if p == "":
1660                 # Depot path does not belong in client.  Must remember
1661                 # this, as previous items should not cause files to
1662                 # exist in this path either.  Remember that the list is
1663                 # being walked from the end, which has higher precedence.
1664                 # Overlap mappings do not exclude previous mappings.
1665                 if not m.overlay:
1666                     paths_filled.append(m.client_side)
1667
1668             else:
1669                 # This mapping matched; no need to search any further.
1670                 # But, the mapping could be rejected if the client path
1671                 # has already been claimed by an earlier mapping (i.e.
1672                 # one later in the list, which we are walking backwards).
1673                 already_mapped_in_client = False
1674                 for f in paths_filled:
1675                     # this is View.Path.match
1676                     if f.match(p):
1677                         already_mapped_in_client = True
1678                         break
1679                 if not already_mapped_in_client:
1680                     # Include this file, unless it is from a line that
1681                     # explicitly said to exclude it.
1682                     if not m.exclude:
1683                         client_path = p
1684
1685                 # a match, even if rejected, always stops the search
1686                 break
1687
1688         return client_path
1689
1690 class P4Sync(Command, P4UserMap):
1691     delete_actions = ( "delete", "move/delete", "purge" )
1692
1693     def __init__(self):
1694         Command.__init__(self)
1695         P4UserMap.__init__(self)
1696         self.options = [
1697                 optparse.make_option("--branch", dest="branch"),
1698                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1699                 optparse.make_option("--changesfile", dest="changesFile"),
1700                 optparse.make_option("--silent", dest="silent", action="store_true"),
1701                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1702                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1703                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1704                                      help="Import into refs/heads/ , not refs/remotes"),
1705                 optparse.make_option("--max-changes", dest="maxChanges"),
1706                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1707                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1708                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1709                                      help="Only sync files that are included in the Perforce Client Spec")
1710         ]
1711         self.description = """Imports from Perforce into a git repository.\n
1712     example:
1713     //depot/my/project/ -- to import the current head
1714     //depot/my/project/@all -- to import everything
1715     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1716
1717     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1718
1719         self.usage += " //depot/path[@revRange]"
1720         self.silent = False
1721         self.createdBranches = set()
1722         self.committedChanges = set()
1723         self.branch = ""
1724         self.detectBranches = False
1725         self.detectLabels = False
1726         self.importLabels = False
1727         self.changesFile = ""
1728         self.syncWithOrigin = True
1729         self.importIntoRemotes = True
1730         self.maxChanges = ""
1731         self.isWindows = (platform.system() == "Windows")
1732         self.keepRepoPath = False
1733         self.depotPaths = None
1734         self.p4BranchesInGit = []
1735         self.cloneExclude = []
1736         self.useClientSpec = False
1737         self.useClientSpec_from_options = False
1738         self.clientSpecDirs = None
1739         self.tempBranches = []
1740         self.tempBranchLocation = "git-p4-tmp"
1741
1742         if gitConfig("git-p4.syncFromOrigin") == "false":
1743             self.syncWithOrigin = False
1744
1745     # Force a checkpoint in fast-import and wait for it to finish
1746     def checkpoint(self):
1747         self.gitStream.write("checkpoint\n\n")
1748         self.gitStream.write("progress checkpoint\n\n")
1749         out = self.gitOutput.readline()
1750         if self.verbose:
1751             print "checkpoint finished: " + out
1752
1753     def extractFilesFromCommit(self, commit):
1754         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1755                              for path in self.cloneExclude]
1756         files = []
1757         fnum = 0
1758         while commit.has_key("depotFile%s" % fnum):
1759             path =  commit["depotFile%s" % fnum]
1760
1761             if [p for p in self.cloneExclude
1762                 if p4PathStartsWith(path, p)]:
1763                 found = False
1764             else:
1765                 found = [p for p in self.depotPaths
1766                          if p4PathStartsWith(path, p)]
1767             if not found:
1768                 fnum = fnum + 1
1769                 continue
1770
1771             file = {}
1772             file["path"] = path
1773             file["rev"] = commit["rev%s" % fnum]
1774             file["action"] = commit["action%s" % fnum]
1775             file["type"] = commit["type%s" % fnum]
1776             files.append(file)
1777             fnum = fnum + 1
1778         return files
1779
1780     def stripRepoPath(self, path, prefixes):
1781         if self.useClientSpec:
1782             return self.clientSpecDirs.map_in_client(path)
1783
1784         if self.keepRepoPath:
1785             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1786
1787         for p in prefixes:
1788             if p4PathStartsWith(path, p):
1789                 path = path[len(p):]
1790
1791         return path
1792
1793     def splitFilesIntoBranches(self, commit):
1794         branches = {}
1795         fnum = 0
1796         while commit.has_key("depotFile%s" % fnum):
1797             path =  commit["depotFile%s" % fnum]
1798             found = [p for p in self.depotPaths
1799                      if p4PathStartsWith(path, p)]
1800             if not found:
1801                 fnum = fnum + 1
1802                 continue
1803
1804             file = {}
1805             file["path"] = path
1806             file["rev"] = commit["rev%s" % fnum]
1807             file["action"] = commit["action%s" % fnum]
1808             file["type"] = commit["type%s" % fnum]
1809             fnum = fnum + 1
1810
1811             relPath = self.stripRepoPath(path, self.depotPaths)
1812             relPath = wildcard_decode(relPath)
1813
1814             for branch in self.knownBranches.keys():
1815
1816                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1817                 if relPath.startswith(branch + "/"):
1818                     if branch not in branches:
1819                         branches[branch] = []
1820                     branches[branch].append(file)
1821                     break
1822
1823         return branches
1824
1825     # output one file from the P4 stream
1826     # - helper for streamP4Files
1827
1828     def streamOneP4File(self, file, contents):
1829         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1830         relPath = wildcard_decode(relPath)
1831         if verbose:
1832             sys.stderr.write("%s\n" % relPath)
1833
1834         (type_base, type_mods) = split_p4_type(file["type"])
1835
1836         git_mode = "100644"
1837         if "x" in type_mods:
1838             git_mode = "100755"
1839         if type_base == "symlink":
1840             git_mode = "120000"
1841             # p4 print on a symlink contains "target\n"; remove the newline
1842             data = ''.join(contents)
1843             contents = [data[:-1]]
1844
1845         if type_base == "utf16":
1846             # p4 delivers different text in the python output to -G
1847             # than it does when using "print -o", or normal p4 client
1848             # operations.  utf16 is converted to ascii or utf8, perhaps.
1849             # But ascii text saved as -t utf16 is completely mangled.
1850             # Invoke print -o to get the real contents.
1851             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1852             contents = [ text ]
1853
1854         if type_base == "apple":
1855             # Apple filetype files will be streamed as a concatenation of
1856             # its appledouble header and the contents.  This is useless
1857             # on both macs and non-macs.  If using "print -q -o xx", it
1858             # will create "xx" with the data, and "%xx" with the header.
1859             # This is also not very useful.
1860             #
1861             # Ideally, someday, this script can learn how to generate
1862             # appledouble files directly and import those to git, but
1863             # non-mac machines can never find a use for apple filetype.
1864             print "\nIgnoring apple filetype file %s" % file['depotFile']
1865             return
1866
1867         # Perhaps windows wants unicode, utf16 newlines translated too;
1868         # but this is not doing it.
1869         if self.isWindows and type_base == "text":
1870             mangled = []
1871             for data in contents:
1872                 data = data.replace("\r\n", "\n")
1873                 mangled.append(data)
1874             contents = mangled
1875
1876         # Note that we do not try to de-mangle keywords on utf16 files,
1877         # even though in theory somebody may want that.
1878         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1879         if pattern:
1880             regexp = re.compile(pattern, re.VERBOSE)
1881             text = ''.join(contents)
1882             text = regexp.sub(r'$\1$', text)
1883             contents = [ text ]
1884
1885         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1886
1887         # total length...
1888         length = 0
1889         for d in contents:
1890             length = length + len(d)
1891
1892         self.gitStream.write("data %d\n" % length)
1893         for d in contents:
1894             self.gitStream.write(d)
1895         self.gitStream.write("\n")
1896
1897     def streamOneP4Deletion(self, file):
1898         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1899         relPath = wildcard_decode(relPath)
1900         if verbose:
1901             sys.stderr.write("delete %s\n" % relPath)
1902         self.gitStream.write("D %s\n" % relPath)
1903
1904     # handle another chunk of streaming data
1905     def streamP4FilesCb(self, marshalled):
1906
1907         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1908             # start of a new file - output the old one first
1909             self.streamOneP4File(self.stream_file, self.stream_contents)
1910             self.stream_file = {}
1911             self.stream_contents = []
1912             self.stream_have_file_info = False
1913
1914         # pick up the new file information... for the
1915         # 'data' field we need to append to our array
1916         for k in marshalled.keys():
1917             if k == 'data':
1918                 self.stream_contents.append(marshalled['data'])
1919             else:
1920                 self.stream_file[k] = marshalled[k]
1921
1922         self.stream_have_file_info = True
1923
1924     # Stream directly from "p4 files" into "git fast-import"
1925     def streamP4Files(self, files):
1926         filesForCommit = []
1927         filesToRead = []
1928         filesToDelete = []
1929
1930         for f in files:
1931             # if using a client spec, only add the files that have
1932             # a path in the client
1933             if self.clientSpecDirs:
1934                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1935                     continue
1936
1937             filesForCommit.append(f)
1938             if f['action'] in self.delete_actions:
1939                 filesToDelete.append(f)
1940             else:
1941                 filesToRead.append(f)
1942
1943         # deleted files...
1944         for f in filesToDelete:
1945             self.streamOneP4Deletion(f)
1946
1947         if len(filesToRead) > 0:
1948             self.stream_file = {}
1949             self.stream_contents = []
1950             self.stream_have_file_info = False
1951
1952             # curry self argument
1953             def streamP4FilesCbSelf(entry):
1954                 self.streamP4FilesCb(entry)
1955
1956             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1957
1958             p4CmdList(["-x", "-", "print"],
1959                       stdin=fileArgs,
1960                       cb=streamP4FilesCbSelf)
1961
1962             # do the last chunk
1963             if self.stream_file.has_key('depotFile'):
1964                 self.streamOneP4File(self.stream_file, self.stream_contents)
1965
1966     def make_email(self, userid):
1967         if userid in self.users:
1968             return self.users[userid]
1969         else:
1970             return "%s <a@b>" % userid
1971
1972     # Stream a p4 tag
1973     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
1974         if verbose:
1975             print "writing tag %s for commit %s" % (labelName, commit)
1976         gitStream.write("tag %s\n" % labelName)
1977         gitStream.write("from %s\n" % commit)
1978
1979         if labelDetails.has_key('Owner'):
1980             owner = labelDetails["Owner"]
1981         else:
1982             owner = None
1983
1984         # Try to use the owner of the p4 label, or failing that,
1985         # the current p4 user id.
1986         if owner:
1987             email = self.make_email(owner)
1988         else:
1989             email = self.make_email(self.p4UserId())
1990         tagger = "%s %s %s" % (email, epoch, self.tz)
1991
1992         gitStream.write("tagger %s\n" % tagger)
1993
1994         print "labelDetails=",labelDetails
1995         if labelDetails.has_key('Description'):
1996             description = labelDetails['Description']
1997         else:
1998             description = 'Label from git p4'
1999
2000         gitStream.write("data %d\n" % len(description))
2001         gitStream.write(description)
2002         gitStream.write("\n")
2003
2004     def commit(self, details, files, branch, branchPrefixes, parent = ""):
2005         epoch = details["time"]
2006         author = details["user"]
2007         self.branchPrefixes = branchPrefixes
2008
2009         if self.verbose:
2010             print "commit into %s" % branch
2011
2012         # start with reading files; if that fails, we should not
2013         # create a commit.
2014         new_files = []
2015         for f in files:
2016             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
2017                 new_files.append (f)
2018             else:
2019                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2020
2021         self.gitStream.write("commit %s\n" % branch)
2022 #        gitStream.write("mark :%s\n" % details["change"])
2023         self.committedChanges.add(int(details["change"]))
2024         committer = ""
2025         if author not in self.users:
2026             self.getUserMapFromPerforceServer()
2027         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2028
2029         self.gitStream.write("committer %s\n" % committer)
2030
2031         self.gitStream.write("data <<EOT\n")
2032         self.gitStream.write(details["desc"])
2033         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2034                              % (','.join (branchPrefixes), details["change"]))
2035         if len(details['options']) > 0:
2036             self.gitStream.write(": options = %s" % details['options'])
2037         self.gitStream.write("]\nEOT\n\n")
2038
2039         if len(parent) > 0:
2040             if self.verbose:
2041                 print "parent %s" % parent
2042             self.gitStream.write("from %s\n" % parent)
2043
2044         self.streamP4Files(new_files)
2045         self.gitStream.write("\n")
2046
2047         change = int(details["change"])
2048
2049         if self.labels.has_key(change):
2050             label = self.labels[change]
2051             labelDetails = label[0]
2052             labelRevisions = label[1]
2053             if self.verbose:
2054                 print "Change %s is labelled %s" % (change, labelDetails)
2055
2056             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2057                                                     for p in branchPrefixes])
2058
2059             if len(files) == len(labelRevisions):
2060
2061                 cleanedFiles = {}
2062                 for info in files:
2063                     if info["action"] in self.delete_actions:
2064                         continue
2065                     cleanedFiles[info["depotFile"]] = info["rev"]
2066
2067                 if cleanedFiles == labelRevisions:
2068                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2069
2070                 else:
2071                     if not self.silent:
2072                         print ("Tag %s does not match with change %s: files do not match."
2073                                % (labelDetails["label"], change))
2074
2075             else:
2076                 if not self.silent:
2077                     print ("Tag %s does not match with change %s: file count is different."
2078                            % (labelDetails["label"], change))
2079
2080     # Build a dictionary of changelists and labels, for "detect-labels" option.
2081     def getLabels(self):
2082         self.labels = {}
2083
2084         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2085         if len(l) > 0 and not self.silent:
2086             print "Finding files belonging to labels in %s" % `self.depotPaths`
2087
2088         for output in l:
2089             label = output["label"]
2090             revisions = {}
2091             newestChange = 0
2092             if self.verbose:
2093                 print "Querying files for label %s" % label
2094             for file in p4CmdList(["files"] +
2095                                       ["%s...@%s" % (p, label)
2096                                           for p in self.depotPaths]):
2097                 revisions[file["depotFile"]] = file["rev"]
2098                 change = int(file["change"])
2099                 if change > newestChange:
2100                     newestChange = change
2101
2102             self.labels[newestChange] = [output, revisions]
2103
2104         if self.verbose:
2105             print "Label changes: %s" % self.labels.keys()
2106
2107     # Import p4 labels as git tags. A direct mapping does not
2108     # exist, so assume that if all the files are at the same revision
2109     # then we can use that, or it's something more complicated we should
2110     # just ignore.
2111     def importP4Labels(self, stream, p4Labels):
2112         if verbose:
2113             print "import p4 labels: " + ' '.join(p4Labels)
2114
2115         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2116         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2117         if len(validLabelRegexp) == 0:
2118             validLabelRegexp = defaultLabelRegexp
2119         m = re.compile(validLabelRegexp)
2120
2121         for name in p4Labels:
2122             commitFound = False
2123
2124             if not m.match(name):
2125                 if verbose:
2126                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2127                 continue
2128
2129             if name in ignoredP4Labels:
2130                 continue
2131
2132             labelDetails = p4CmdList(['label', "-o", name])[0]
2133
2134             # get the most recent changelist for each file in this label
2135             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2136                                 for p in self.depotPaths])
2137
2138             if change.has_key('change'):
2139                 # find the corresponding git commit; take the oldest commit
2140                 changelist = int(change['change'])
2141                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2142                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2143                 if len(gitCommit) == 0:
2144                     print "could not find git commit for changelist %d" % changelist
2145                 else:
2146                     gitCommit = gitCommit.strip()
2147                     commitFound = True
2148                     # Convert from p4 time format
2149                     try:
2150                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2151                     except ValueError:
2152                         print "Could not convert label time %s" % labelDetail['Update']
2153                         tmwhen = 1
2154
2155                     when = int(time.mktime(tmwhen))
2156                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2157                     if verbose:
2158                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2159             else:
2160                 if verbose:
2161                     print "Label %s has no changelists - possibly deleted?" % name
2162
2163             if not commitFound:
2164                 # We can't import this label; don't try again as it will get very
2165                 # expensive repeatedly fetching all the files for labels that will
2166                 # never be imported. If the label is moved in the future, the
2167                 # ignore will need to be removed manually.
2168                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2169
2170     def guessProjectName(self):
2171         for p in self.depotPaths:
2172             if p.endswith("/"):
2173                 p = p[:-1]
2174             p = p[p.strip().rfind("/") + 1:]
2175             if not p.endswith("/"):
2176                p += "/"
2177             return p
2178
2179     def getBranchMapping(self):
2180         lostAndFoundBranches = set()
2181
2182         user = gitConfig("git-p4.branchUser")
2183         if len(user) > 0:
2184             command = "branches -u %s" % user
2185         else:
2186             command = "branches"
2187
2188         for info in p4CmdList(command):
2189             details = p4Cmd(["branch", "-o", info["branch"]])
2190             viewIdx = 0
2191             while details.has_key("View%s" % viewIdx):
2192                 paths = details["View%s" % viewIdx].split(" ")
2193                 viewIdx = viewIdx + 1
2194                 # require standard //depot/foo/... //depot/bar/... mapping
2195                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2196                     continue
2197                 source = paths[0]
2198                 destination = paths[1]
2199                 ## HACK
2200                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2201                     source = source[len(self.depotPaths[0]):-4]
2202                     destination = destination[len(self.depotPaths[0]):-4]
2203
2204                     if destination in self.knownBranches:
2205                         if not self.silent:
2206                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2207                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2208                         continue
2209
2210                     self.knownBranches[destination] = source
2211
2212                     lostAndFoundBranches.discard(destination)
2213
2214                     if source not in self.knownBranches:
2215                         lostAndFoundBranches.add(source)
2216
2217         # Perforce does not strictly require branches to be defined, so we also
2218         # check git config for a branch list.
2219         #
2220         # Example of branch definition in git config file:
2221         # [git-p4]
2222         #   branchList=main:branchA
2223         #   branchList=main:branchB
2224         #   branchList=branchA:branchC
2225         configBranches = gitConfigList("git-p4.branchList")
2226         for branch in configBranches:
2227             if branch:
2228                 (source, destination) = branch.split(":")
2229                 self.knownBranches[destination] = source
2230
2231                 lostAndFoundBranches.discard(destination)
2232
2233                 if source not in self.knownBranches:
2234                     lostAndFoundBranches.add(source)
2235
2236
2237         for branch in lostAndFoundBranches:
2238             self.knownBranches[branch] = branch
2239
2240     def getBranchMappingFromGitBranches(self):
2241         branches = p4BranchesInGit(self.importIntoRemotes)
2242         for branch in branches.keys():
2243             if branch == "master":
2244                 branch = "main"
2245             else:
2246                 branch = branch[len(self.projectName):]
2247             self.knownBranches[branch] = branch
2248
2249     def listExistingP4GitBranches(self):
2250         # branches holds mapping from name to commit
2251         branches = p4BranchesInGit(self.importIntoRemotes)
2252         self.p4BranchesInGit = branches.keys()
2253         for branch in branches.keys():
2254             self.initialParents[self.refPrefix + branch] = branches[branch]
2255
2256     def updateOptionDict(self, d):
2257         option_keys = {}
2258         if self.keepRepoPath:
2259             option_keys['keepRepoPath'] = 1
2260
2261         d["options"] = ' '.join(sorted(option_keys.keys()))
2262
2263     def readOptions(self, d):
2264         self.keepRepoPath = (d.has_key('options')
2265                              and ('keepRepoPath' in d['options']))
2266
2267     def gitRefForBranch(self, branch):
2268         if branch == "main":
2269             return self.refPrefix + "master"
2270
2271         if len(branch) <= 0:
2272             return branch
2273
2274         return self.refPrefix + self.projectName + branch
2275
2276     def gitCommitByP4Change(self, ref, change):
2277         if self.verbose:
2278             print "looking in ref " + ref + " for change %s using bisect..." % change
2279
2280         earliestCommit = ""
2281         latestCommit = parseRevision(ref)
2282
2283         while True:
2284             if self.verbose:
2285                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2286             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2287             if len(next) == 0:
2288                 if self.verbose:
2289                     print "argh"
2290                 return ""
2291             log = extractLogMessageFromGitCommit(next)
2292             settings = extractSettingsGitLog(log)
2293             currentChange = int(settings['change'])
2294             if self.verbose:
2295                 print "current change %s" % currentChange
2296
2297             if currentChange == change:
2298                 if self.verbose:
2299                     print "found %s" % next
2300                 return next
2301
2302             if currentChange < change:
2303                 earliestCommit = "^%s" % next
2304             else:
2305                 latestCommit = "%s" % next
2306
2307         return ""
2308
2309     def importNewBranch(self, branch, maxChange):
2310         # make fast-import flush all changes to disk and update the refs using the checkpoint
2311         # command so that we can try to find the branch parent in the git history
2312         self.gitStream.write("checkpoint\n\n");
2313         self.gitStream.flush();
2314         branchPrefix = self.depotPaths[0] + branch + "/"
2315         range = "@1,%s" % maxChange
2316         #print "prefix" + branchPrefix
2317         changes = p4ChangesForPaths([branchPrefix], range)
2318         if len(changes) <= 0:
2319             return False
2320         firstChange = changes[0]
2321         #print "first change in branch: %s" % firstChange
2322         sourceBranch = self.knownBranches[branch]
2323         sourceDepotPath = self.depotPaths[0] + sourceBranch
2324         sourceRef = self.gitRefForBranch(sourceBranch)
2325         #print "source " + sourceBranch
2326
2327         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2328         #print "branch parent: %s" % branchParentChange
2329         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2330         if len(gitParent) > 0:
2331             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2332             #print "parent git commit: %s" % gitParent
2333
2334         self.importChanges(changes)
2335         return True
2336
2337     def searchParent(self, parent, branch, target):
2338         parentFound = False
2339         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2340             blob = blob.strip()
2341             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2342                 parentFound = True
2343                 if self.verbose:
2344                     print "Found parent of %s in commit %s" % (branch, blob)
2345                 break
2346         if parentFound:
2347             return blob
2348         else:
2349             return None
2350
2351     def importChanges(self, changes):
2352         cnt = 1
2353         for change in changes:
2354             description = p4Cmd(["describe", str(change)])
2355             self.updateOptionDict(description)
2356
2357             if not self.silent:
2358                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2359                 sys.stdout.flush()
2360             cnt = cnt + 1
2361
2362             try:
2363                 if self.detectBranches:
2364                     branches = self.splitFilesIntoBranches(description)
2365                     for branch in branches.keys():
2366                         ## HACK  --hwn
2367                         branchPrefix = self.depotPaths[0] + branch + "/"
2368
2369                         parent = ""
2370
2371                         filesForCommit = branches[branch]
2372
2373                         if self.verbose:
2374                             print "branch is %s" % branch
2375
2376                         self.updatedBranches.add(branch)
2377
2378                         if branch not in self.createdBranches:
2379                             self.createdBranches.add(branch)
2380                             parent = self.knownBranches[branch]
2381                             if parent == branch:
2382                                 parent = ""
2383                             else:
2384                                 fullBranch = self.projectName + branch
2385                                 if fullBranch not in self.p4BranchesInGit:
2386                                     if not self.silent:
2387                                         print("\n    Importing new branch %s" % fullBranch);
2388                                     if self.importNewBranch(branch, change - 1):
2389                                         parent = ""
2390                                         self.p4BranchesInGit.append(fullBranch)
2391                                     if not self.silent:
2392                                         print("\n    Resuming with change %s" % change);
2393
2394                                 if self.verbose:
2395                                     print "parent determined through known branches: %s" % parent
2396
2397                         branch = self.gitRefForBranch(branch)
2398                         parent = self.gitRefForBranch(parent)
2399
2400                         if self.verbose:
2401                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2402
2403                         if len(parent) == 0 and branch in self.initialParents:
2404                             parent = self.initialParents[branch]
2405                             del self.initialParents[branch]
2406
2407                         blob = None
2408                         if len(parent) > 0:
2409                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2410                             if self.verbose:
2411                                 print "Creating temporary branch: " + tempBranch
2412                             self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2413                             self.tempBranches.append(tempBranch)
2414                             self.checkpoint()
2415                             blob = self.searchParent(parent, branch, tempBranch)
2416                         if blob:
2417                             self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2418                         else:
2419                             if self.verbose:
2420                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2421                             self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2422                 else:
2423                     files = self.extractFilesFromCommit(description)
2424                     self.commit(description, files, self.branch, self.depotPaths,
2425                                 self.initialParent)
2426                     self.initialParent = ""
2427             except IOError:
2428                 print self.gitError.read()
2429                 sys.exit(1)
2430
2431     def importHeadRevision(self, revision):
2432         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2433
2434         details = {}
2435         details["user"] = "git perforce import user"
2436         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2437                            % (' '.join(self.depotPaths), revision))
2438         details["change"] = revision
2439         newestRevision = 0
2440
2441         fileCnt = 0
2442         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2443
2444         for info in p4CmdList(["files"] + fileArgs):
2445
2446             if 'code' in info and info['code'] == 'error':
2447                 sys.stderr.write("p4 returned an error: %s\n"
2448                                  % info['data'])
2449                 if info['data'].find("must refer to client") >= 0:
2450                     sys.stderr.write("This particular p4 error is misleading.\n")
2451                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2452                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2453                 sys.exit(1)
2454             if 'p4ExitCode' in info:
2455                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2456                 sys.exit(1)
2457
2458
2459             change = int(info["change"])
2460             if change > newestRevision:
2461                 newestRevision = change
2462
2463             if info["action"] in self.delete_actions:
2464                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2465                 #fileCnt = fileCnt + 1
2466                 continue
2467
2468             for prop in ["depotFile", "rev", "action", "type" ]:
2469                 details["%s%s" % (prop, fileCnt)] = info[prop]
2470
2471             fileCnt = fileCnt + 1
2472
2473         details["change"] = newestRevision
2474
2475         # Use time from top-most change so that all git p4 clones of
2476         # the same p4 repo have the same commit SHA1s.
2477         res = p4CmdList("describe -s %d" % newestRevision)
2478         newestTime = None
2479         for r in res:
2480             if r.has_key('time'):
2481                 newestTime = int(r['time'])
2482         if newestTime is None:
2483             die("\"describe -s\" on newest change %d did not give a time")
2484         details["time"] = newestTime
2485
2486         self.updateOptionDict(details)
2487         try:
2488             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2489         except IOError:
2490             print "IO error with git fast-import. Is your git version recent enough?"
2491             print self.gitError.read()
2492
2493
2494     def run(self, args):
2495         self.depotPaths = []
2496         self.changeRange = ""
2497         self.initialParent = ""
2498         self.previousDepotPaths = []
2499
2500         # map from branch depot path to parent branch
2501         self.knownBranches = {}
2502         self.initialParents = {}
2503         self.hasOrigin = originP4BranchesExist()
2504         if not self.syncWithOrigin:
2505             self.hasOrigin = False
2506
2507         if self.importIntoRemotes:
2508             self.refPrefix = "refs/remotes/p4/"
2509         else:
2510             self.refPrefix = "refs/heads/p4/"
2511
2512         if self.syncWithOrigin and self.hasOrigin:
2513             if not self.silent:
2514                 print "Syncing with origin first by calling git fetch origin"
2515             system("git fetch origin")
2516
2517         if len(self.branch) == 0:
2518             self.branch = self.refPrefix + "master"
2519             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2520                 system("git update-ref %s refs/heads/p4" % self.branch)
2521                 system("git branch -D p4");
2522             # create it /after/ importing, when master exists
2523             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2524                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2525
2526         # accept either the command-line option, or the configuration variable
2527         if self.useClientSpec:
2528             # will use this after clone to set the variable
2529             self.useClientSpec_from_options = True
2530         else:
2531             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2532                 self.useClientSpec = True
2533         if self.useClientSpec:
2534             self.clientSpecDirs = getClientSpec()
2535
2536         # TODO: should always look at previous commits,
2537         # merge with previous imports, if possible.
2538         if args == []:
2539             if self.hasOrigin:
2540                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2541             self.listExistingP4GitBranches()
2542
2543             if len(self.p4BranchesInGit) > 1:
2544                 if not self.silent:
2545                     print "Importing from/into multiple branches"
2546                 self.detectBranches = True
2547
2548             if self.verbose:
2549                 print "branches: %s" % self.p4BranchesInGit
2550
2551             p4Change = 0
2552             for branch in self.p4BranchesInGit:
2553                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2554
2555                 settings = extractSettingsGitLog(logMsg)
2556
2557                 self.readOptions(settings)
2558                 if (settings.has_key('depot-paths')
2559                     and settings.has_key ('change')):
2560                     change = int(settings['change']) + 1
2561                     p4Change = max(p4Change, change)
2562
2563                     depotPaths = sorted(settings['depot-paths'])
2564                     if self.previousDepotPaths == []:
2565                         self.previousDepotPaths = depotPaths
2566                     else:
2567                         paths = []
2568                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2569                             prev_list = prev.split("/")
2570                             cur_list = cur.split("/")
2571                             for i in range(0, min(len(cur_list), len(prev_list))):
2572                                 if cur_list[i] <> prev_list[i]:
2573                                     i = i - 1
2574                                     break
2575
2576                             paths.append ("/".join(cur_list[:i + 1]))
2577
2578                         self.previousDepotPaths = paths
2579
2580             if p4Change > 0:
2581                 self.depotPaths = sorted(self.previousDepotPaths)
2582                 self.changeRange = "@%s,#head" % p4Change
2583                 if not self.detectBranches:
2584                     self.initialParent = parseRevision(self.branch)
2585                 if not self.silent and not self.detectBranches:
2586                     print "Performing incremental import into %s git branch" % self.branch
2587
2588         if not self.branch.startswith("refs/"):
2589             self.branch = "refs/heads/" + self.branch
2590
2591         if len(args) == 0 and self.depotPaths:
2592             if not self.silent:
2593                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2594         else:
2595             if self.depotPaths and self.depotPaths != args:
2596                 print ("previous import used depot path %s and now %s was specified. "
2597                        "This doesn't work!" % (' '.join (self.depotPaths),
2598                                                ' '.join (args)))
2599                 sys.exit(1)
2600
2601             self.depotPaths = sorted(args)
2602
2603         revision = ""
2604         self.users = {}
2605
2606         # Make sure no revision specifiers are used when --changesfile
2607         # is specified.
2608         bad_changesfile = False
2609         if len(self.changesFile) > 0:
2610             for p in self.depotPaths:
2611                 if p.find("@") >= 0 or p.find("#") >= 0:
2612                     bad_changesfile = True
2613                     break
2614         if bad_changesfile:
2615             die("Option --changesfile is incompatible with revision specifiers")
2616
2617         newPaths = []
2618         for p in self.depotPaths:
2619             if p.find("@") != -1:
2620                 atIdx = p.index("@")
2621                 self.changeRange = p[atIdx:]
2622                 if self.changeRange == "@all":
2623                     self.changeRange = ""
2624                 elif ',' not in self.changeRange:
2625                     revision = self.changeRange
2626                     self.changeRange = ""
2627                 p = p[:atIdx]
2628             elif p.find("#") != -1:
2629                 hashIdx = p.index("#")
2630                 revision = p[hashIdx:]
2631                 p = p[:hashIdx]
2632             elif self.previousDepotPaths == []:
2633                 # pay attention to changesfile, if given, else import
2634                 # the entire p4 tree at the head revision
2635                 if len(self.changesFile) == 0:
2636                     revision = "#head"
2637
2638             p = re.sub ("\.\.\.$", "", p)
2639             if not p.endswith("/"):
2640                 p += "/"
2641
2642             newPaths.append(p)
2643
2644         self.depotPaths = newPaths
2645
2646         self.loadUserMapFromCache()
2647         self.labels = {}
2648         if self.detectLabels:
2649             self.getLabels();
2650
2651         if self.detectBranches:
2652             ## FIXME - what's a P4 projectName ?
2653             self.projectName = self.guessProjectName()
2654
2655             if self.hasOrigin:
2656                 self.getBranchMappingFromGitBranches()
2657             else:
2658                 self.getBranchMapping()
2659             if self.verbose:
2660                 print "p4-git branches: %s" % self.p4BranchesInGit
2661                 print "initial parents: %s" % self.initialParents
2662             for b in self.p4BranchesInGit:
2663                 if b != "master":
2664
2665                     ## FIXME
2666                     b = b[len(self.projectName):]
2667                 self.createdBranches.add(b)
2668
2669         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2670
2671         importProcess = subprocess.Popen(["git", "fast-import"],
2672                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2673                                          stderr=subprocess.PIPE);
2674         self.gitOutput = importProcess.stdout
2675         self.gitStream = importProcess.stdin
2676         self.gitError = importProcess.stderr
2677
2678         if revision:
2679             self.importHeadRevision(revision)
2680         else:
2681             changes = []
2682
2683             if len(self.changesFile) > 0:
2684                 output = open(self.changesFile).readlines()
2685                 changeSet = set()
2686                 for line in output:
2687                     changeSet.add(int(line))
2688
2689                 for change in changeSet:
2690                     changes.append(change)
2691
2692                 changes.sort()
2693             else:
2694                 # catch "git p4 sync" with no new branches, in a repo that
2695                 # does not have any existing p4 branches
2696                 if len(args) == 0 and not self.p4BranchesInGit:
2697                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2698                 if self.verbose:
2699                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2700                                                               self.changeRange)
2701                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2702
2703                 if len(self.maxChanges) > 0:
2704                     changes = changes[:min(int(self.maxChanges), len(changes))]
2705
2706             if len(changes) == 0:
2707                 if not self.silent:
2708                     print "No changes to import!"
2709             else:
2710                 if not self.silent and not self.detectBranches:
2711                     print "Import destination: %s" % self.branch
2712
2713                 self.updatedBranches = set()
2714
2715                 self.importChanges(changes)
2716
2717                 if not self.silent:
2718                     print ""
2719                     if len(self.updatedBranches) > 0:
2720                         sys.stdout.write("Updated branches: ")
2721                         for b in self.updatedBranches:
2722                             sys.stdout.write("%s " % b)
2723                         sys.stdout.write("\n")
2724
2725         if gitConfig("git-p4.importLabels", "--bool") == "true":
2726             self.importLabels = True
2727
2728         if self.importLabels:
2729             p4Labels = getP4Labels(self.depotPaths)
2730             gitTags = getGitTags()
2731
2732             missingP4Labels = p4Labels - gitTags
2733             self.importP4Labels(self.gitStream, missingP4Labels)
2734
2735         self.gitStream.close()
2736         if importProcess.wait() != 0:
2737             die("fast-import failed: %s" % self.gitError.read())
2738         self.gitOutput.close()
2739         self.gitError.close()
2740
2741         # Cleanup temporary branches created during import
2742         if self.tempBranches != []:
2743             for branch in self.tempBranches:
2744                 read_pipe("git update-ref -d %s" % branch)
2745             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2746
2747         return True
2748
2749 class P4Rebase(Command):
2750     def __init__(self):
2751         Command.__init__(self)
2752         self.options = [
2753                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2754         ]
2755         self.importLabels = False
2756         self.description = ("Fetches the latest revision from perforce and "
2757                             + "rebases the current work (branch) against it")
2758
2759     def run(self, args):
2760         sync = P4Sync()
2761         sync.importLabels = self.importLabels
2762         sync.run([])
2763
2764         return self.rebase()
2765
2766     def rebase(self):
2767         if os.system("git update-index --refresh") != 0:
2768             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2769         if len(read_pipe("git diff-index HEAD --")) > 0:
2770             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2771
2772         [upstream, settings] = findUpstreamBranchPoint()
2773         if len(upstream) == 0:
2774             die("Cannot find upstream branchpoint for rebase")
2775
2776         # the branchpoint may be p4/foo~3, so strip off the parent
2777         upstream = re.sub("~[0-9]+$", "", upstream)
2778
2779         print "Rebasing the current branch onto %s" % upstream
2780         oldHead = read_pipe("git rev-parse HEAD").strip()
2781         system("git rebase %s" % upstream)
2782         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2783         return True
2784
2785 class P4Clone(P4Sync):
2786     def __init__(self):
2787         P4Sync.__init__(self)
2788         self.description = "Creates a new git repository and imports from Perforce into it"
2789         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2790         self.options += [
2791             optparse.make_option("--destination", dest="cloneDestination",
2792                                  action='store', default=None,
2793                                  help="where to leave result of the clone"),
2794             optparse.make_option("-/", dest="cloneExclude",
2795                                  action="append", type="string",
2796                                  help="exclude depot path"),
2797             optparse.make_option("--bare", dest="cloneBare",
2798                                  action="store_true", default=False),
2799         ]
2800         self.cloneDestination = None
2801         self.needsGit = False
2802         self.cloneBare = False
2803
2804     # This is required for the "append" cloneExclude action
2805     def ensure_value(self, attr, value):
2806         if not hasattr(self, attr) or getattr(self, attr) is None:
2807             setattr(self, attr, value)
2808         return getattr(self, attr)
2809
2810     def defaultDestination(self, args):
2811         ## TODO: use common prefix of args?
2812         depotPath = args[0]
2813         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2814         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2815         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2816         depotDir = re.sub(r"/$", "", depotDir)
2817         return os.path.split(depotDir)[1]
2818
2819     def run(self, args):
2820         if len(args) < 1:
2821             return False
2822
2823         if self.keepRepoPath and not self.cloneDestination:
2824             sys.stderr.write("Must specify destination for --keep-path\n")
2825             sys.exit(1)
2826
2827         depotPaths = args
2828
2829         if not self.cloneDestination and len(depotPaths) > 1:
2830             self.cloneDestination = depotPaths[-1]
2831             depotPaths = depotPaths[:-1]
2832
2833         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2834         for p in depotPaths:
2835             if not p.startswith("//"):
2836                 return False
2837
2838         if not self.cloneDestination:
2839             self.cloneDestination = self.defaultDestination(args)
2840
2841         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2842
2843         if not os.path.exists(self.cloneDestination):
2844             os.makedirs(self.cloneDestination)
2845         chdir(self.cloneDestination)
2846
2847         init_cmd = [ "git", "init" ]
2848         if self.cloneBare:
2849             init_cmd.append("--bare")
2850         subprocess.check_call(init_cmd)
2851
2852         if not P4Sync.run(self, depotPaths):
2853             return False
2854         if self.branch != "master":
2855             if self.importIntoRemotes:
2856                 masterbranch = "refs/remotes/p4/master"
2857             else:
2858                 masterbranch = "refs/heads/p4/master"
2859             if gitBranchExists(masterbranch):
2860                 system("git branch master %s" % masterbranch)
2861                 if not self.cloneBare:
2862                     system("git checkout -f")
2863             else:
2864                 print "Could not detect main branch. No checkout/master branch created."
2865
2866         # auto-set this variable if invoked with --use-client-spec
2867         if self.useClientSpec_from_options:
2868             system("git config --bool git-p4.useclientspec true")
2869
2870         return True
2871
2872 class P4Branches(Command):
2873     def __init__(self):
2874         Command.__init__(self)
2875         self.options = [ ]
2876         self.description = ("Shows the git branches that hold imports and their "
2877                             + "corresponding perforce depot paths")
2878         self.verbose = False
2879
2880     def run(self, args):
2881         if originP4BranchesExist():
2882             createOrUpdateBranchesFromOrigin()
2883
2884         cmdline = "git rev-parse --symbolic "
2885         cmdline += " --remotes"
2886
2887         for line in read_pipe_lines(cmdline):
2888             line = line.strip()
2889
2890             if not line.startswith('p4/') or line == "p4/HEAD":
2891                 continue
2892             branch = line
2893
2894             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2895             settings = extractSettingsGitLog(log)
2896
2897             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2898         return True
2899
2900 class HelpFormatter(optparse.IndentedHelpFormatter):
2901     def __init__(self):
2902         optparse.IndentedHelpFormatter.__init__(self)
2903
2904     def format_description(self, description):
2905         if description:
2906             return description + "\n"
2907         else:
2908             return ""
2909
2910 def printUsage(commands):
2911     print "usage: %s <command> [options]" % sys.argv[0]
2912     print ""
2913     print "valid commands: %s" % ", ".join(commands)
2914     print ""
2915     print "Try %s <command> --help for command specific help." % sys.argv[0]
2916     print ""
2917
2918 commands = {
2919     "debug" : P4Debug,
2920     "submit" : P4Submit,
2921     "commit" : P4Submit,
2922     "sync" : P4Sync,
2923     "rebase" : P4Rebase,
2924     "clone" : P4Clone,
2925     "rollback" : P4RollBack,
2926     "branches" : P4Branches
2927 }
2928
2929
2930 def main():
2931     if len(sys.argv[1:]) == 0:
2932         printUsage(commands.keys())
2933         sys.exit(2)
2934
2935     cmd = ""
2936     cmdName = sys.argv[1]
2937     try:
2938         klass = commands[cmdName]
2939         cmd = klass()
2940     except KeyError:
2941         print "unknown command %s" % cmdName
2942         print ""
2943         printUsage(commands.keys())
2944         sys.exit(2)
2945
2946     options = cmd.options
2947     cmd.gitdir = os.environ.get("GIT_DIR", None)
2948
2949     args = sys.argv[2:]
2950
2951     options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))
2952     if cmd.needsGit:
2953         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2954
2955     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2956                                    options,
2957                                    description = cmd.description,
2958                                    formatter = HelpFormatter())
2959
2960     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2961     global verbose
2962     verbose = cmd.verbose
2963     if cmd.needsGit:
2964         if cmd.gitdir == None:
2965             cmd.gitdir = os.path.abspath(".git")
2966             if not isValidGitDir(cmd.gitdir):
2967                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2968                 if os.path.exists(cmd.gitdir):
2969                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2970                     if len(cdup) > 0:
2971                         chdir(cdup);
2972
2973         if not isValidGitDir(cmd.gitdir):
2974             if isValidGitDir(cmd.gitdir + "/.git"):
2975                 cmd.gitdir += "/.git"
2976             else:
2977                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2978
2979         os.environ["GIT_DIR"] = cmd.gitdir
2980
2981     if not cmd.run(args):
2982         parser.print_help()
2983         sys.exit(2)
2984
2985
2986 if __name__ == '__main__':
2987     main()