3 # Copyright 2007-2009 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 '''Mercurial interface to codereview.appspot.com.
19 To configure, set the following options in
20 your repository's .hg/hgrc file.
23 codereview = path/to/codereview.py
26 server = codereview.appspot.com
28 The server should be running Rietveld; see http://code.google.com/p/rietveld/.
30 In addition to the new commands, this extension introduces
31 the file pattern syntax @nnnnnn, where nnnnnn is a change list
32 number, to mean the files included in that change list, which
33 must be associated with the current client.
35 For example, if change 123456 contains the files x.go and y.go,
36 "hg diff @123456" is equivalent to"hg diff x.go y.go".
39 from mercurial import cmdutil, commands, hg, util, error, match
40 from mercurial.node import nullrev, hex, nullid, short
45 from HTMLParser import HTMLParser
47 from xml.etree import ElementTree as ET
49 from elementtree import ElementTree as ET
52 hgversion = util.version()
54 from mercurial.version import version as v
55 hgversion = v.get_version()
58 from mercurial.discovery import findcommonincoming
60 def findcommonincoming(repo, remote):
61 return repo.findcommonincoming(remote)
64 The code review extension requires Mercurial 1.3 or newer.
66 To install a new Mercurial,
68 sudo easy_install mercurial
70 works on most systems.
74 You may need to clear your current Mercurial installation by running:
76 sudo apt-get remove mercurial mercurial-common
77 sudo rm -rf /etc/mercurial
82 if os.access("/etc/mercurial", 0):
86 def promptyesno(ui, msg):
87 # Arguments to ui.prompt changed between 1.3 and 1.3.1.
88 # Even so, some 1.3.1 distributions seem to have the old prompt!?!?
89 # What a terrible way to maintain software.
91 return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0
92 except AttributeError:
93 return ui.prompt(msg, ["&yes", "&no"], "y") != "n"
95 # To experiment with Mercurial in the python interpreter:
96 # >>> repo = hg.repository(ui.ui(), path = ".")
98 #######################################################################
99 # Normally I would split this into multiple files, but it simplifies
100 # import path headaches to keep it all in one file. Sorry.
103 if __name__ == "__main__":
104 print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
107 server = "codereview.appspot.com"
108 server_url_base = None
112 #######################################################################
113 # Change list parsing.
115 # Change lists are stored in .hg/codereview/cl.nnnnnn
116 # where nnnnnn is the number assigned by the code review server.
117 # Most data about a change list is stored on the code review server
118 # too: the description, reviewer, and cc list are all stored there.
119 # The only thing in the cl.nnnnnn file is the list of relevant files.
120 # Also, the existence of the cl.nnnnnn file marks this repository
121 # as the one where the change list lives.
123 emptydiff = """Index: ~rietveld~placeholder~
124 ===================================================================
125 diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
131 def __init__(self, name):
140 self.copied_from = None # None means current user
147 s += "Author: " + cl.copied_from + "\n\n"
148 s += "Mailed: " + str(self.mailed) + "\n"
149 s += "Description:\n"
150 s += Indent(cl.desc, "\t")
156 def EditorText(self):
161 s += "Author: " + cl.copied_from + "\n"
163 s += 'URL: ' + cl.url + ' # cannot edit\n\n'
164 s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
165 s += "CC: " + JoinComma(cl.cc) + "\n"
167 s += "Description:\n"
169 s += "\t<enter description here>\n"
171 s += Indent(cl.desc, "\t")
173 if cl.local or cl.name == "new":
180 def PendingText(self):
182 s = cl.name + ":" + "\n"
183 s += Indent(cl.desc, "\t")
186 s += "\tAuthor: " + cl.copied_from + "\n"
187 s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
188 s += "\tCC: " + JoinComma(cl.cc) + "\n"
191 s += "\t\t" + f + "\n"
194 def Flush(self, ui, repo):
195 if self.name == "new":
196 self.Upload(ui, repo, gofmt_just_warn=True)
197 dir = CodeReviewDir(ui, repo)
198 path = dir + '/cl.' + self.name
199 f = open(path+'!', "w")
200 f.write(self.DiskText())
202 if sys.platform == "win32" and os.path.isfile(path):
204 os.rename(path+'!', path)
205 if self.web and not self.copied_from:
206 EditDesc(self.name, desc=self.desc,
207 reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc))
209 def Delete(self, ui, repo):
210 dir = CodeReviewDir(ui, repo)
211 os.unlink(dir + "/cl." + self.name)
217 if self.name != "new":
218 s = "code review %s: %s" % (self.name, s)
221 def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False):
223 ui.warn("no files in change list\n")
224 if ui.configbool("codereview", "force_gofmt", True) and gofmt:
225 CheckGofmt(ui, repo, self.files, just_warn=gofmt_just_warn)
228 ("content_upload", "1"),
229 ("reviewers", JoinComma(self.reviewer)),
230 ("cc", JoinComma(self.cc)),
231 ("description", self.desc),
233 # Would prefer not to change the subject
234 # on reupload, but /upload requires it.
235 ("subject", self.Subject()),
238 # NOTE(rsc): This duplicates too much of RealMain,
239 # but RealMain doesn't have the most reusable interface.
240 if self.name != "new":
241 form_fields.append(("issue", self.name))
244 vcs = GuessVCS(upload_options)
245 data = vcs.GenerateDiff(self.files)
246 files = vcs.GetBaseFiles(data)
247 if len(data) > MAX_UPLOAD_SIZE:
248 uploaded_diff_file = []
249 form_fields.append(("separate_patches", "1"))
251 uploaded_diff_file = [("data", "data.diff", data)]
253 uploaded_diff_file = [("data", "data.diff", emptydiff)]
254 ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
255 response_body = MySend("/upload", body, content_type=ctype)
258 lines = msg.splitlines()
261 patchset = lines[1].strip()
262 patches = [x.split(" ", 1) for x in lines[2:]]
263 ui.status(msg + "\n")
264 if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
265 raise util.Abort("failed to update issue: " + response_body)
266 issue = msg[msg.rfind("/")+1:]
269 self.url = server_url_base + self.name
270 if not uploaded_diff_file:
271 patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
273 vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
275 MySend("/" + issue + "/mail", payload="")
280 def Mail(self, ui,repo):
281 pmsg = "Hello " + JoinComma(self.reviewer)
283 pmsg += " (cc: %s)" % (', '.join(self.cc),)
287 pmsg += "I'd like you to review this change.\n"
289 pmsg += "Please take another look.\n"
290 PostMessage(ui, self.name, pmsg, subject=self.Subject())
294 def GoodCLName(name):
295 return re.match("^[0-9]+$", name)
297 def ParseCL(text, name):
309 for line in text.split('\n'):
312 if line != '' and line[0] == '#':
314 if line == '' or line[0] == ' ' or line[0] == '\t':
315 if sname == None and line != '':
316 return None, lineno, 'text outside section'
318 sections[sname] += line + '\n'
322 s, val = line[:p].strip(), line[p+1:].strip()
326 sections[sname] += val + '\n'
328 return None, lineno, 'malformed section header'
331 sections[k] = StripCommon(sections[k]).rstrip()
334 if sections['Author']:
335 cl.copied_from = sections['Author']
336 cl.desc = sections['Description']
337 for line in sections['Files'].split('\n'):
340 line = line[0:i].rstrip()
343 cl.files.append(line)
344 cl.reviewer = SplitCommaSpace(sections['Reviewer'])
345 cl.cc = SplitCommaSpace(sections['CC'])
346 cl.url = sections['URL']
347 if sections['Mailed'] != 'False':
348 # Odd default, but avoids spurious mailings when
349 # reading old CLs that do not have a Mailed: line.
350 # CLs created with this update will always have
351 # Mailed: False on disk.
353 if cl.desc == '<enter description here>':
357 def SplitCommaSpace(s):
361 return re.split(", *", s)
372 def ExceptionDetail():
373 s = str(sys.exc_info()[0])
374 if s.startswith("<type '") and s.endswith("'>"):
376 elif s.startswith("<class '") and s.endswith("'>"):
378 arg = str(sys.exc_info()[1])
383 def IsLocalCL(ui, repo, name):
384 return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
386 # Load CL from disk and/or the web.
387 def LoadCL(ui, repo, name, web=True):
388 if not GoodCLName(name):
389 return None, "invalid CL name"
390 dir = CodeReviewDir(ui, repo)
391 path = dir + "cl." + name
392 if os.access(path, 0):
396 cl, lineno, err = ParseCL(text, name)
398 return None, "malformed CL data: "+err
404 f = GetSettings(name)
406 return None, "cannot load CL %s from code review server: %s" % (name, ExceptionDetail())
407 if 'reviewers' not in f:
408 return None, "malformed response loading CL data from code review server"
409 cl.reviewer = SplitCommaSpace(f['reviewers'])
410 cl.cc = SplitCommaSpace(f['cc'])
411 if cl.local and cl.copied_from and cl.desc:
412 # local copy of CL written by someone else
413 # and we saved a description. use that one,
414 # so that committers can edit the description
415 # before doing hg submit.
418 cl.desc = f['description']
419 cl.url = server_url_base + name
423 class LoadCLThread(threading.Thread):
424 def __init__(self, ui, repo, dir, f, web):
425 threading.Thread.__init__(self)
433 cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
435 self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
439 # Load all the CLs from this repository.
440 def LoadAllCL(ui, repo, web=True):
441 dir = CodeReviewDir(ui, repo)
443 files = [f for f in os.listdir(dir) if f.startswith('cl.')]
449 t = LoadCLThread(ui, repo, dir, f, web)
452 # first request: wait in case it needs to authenticate
453 # otherwise we get lots of user/password prompts
454 # running in parallel.
467 # Find repository root. On error, ui.warn and return None
468 def RepoDir(ui, repo):
470 if not url.startswith('file:'):
471 ui.warn("repository %s is not in local file system\n" % (url,))
474 if url.endswith('/'):
478 # Find (or make) code review directory. On error, ui.warn and return None
479 def CodeReviewDir(ui, repo):
480 dir = RepoDir(ui, repo)
483 dir += '/.hg/codereview/'
484 if not os.path.isdir(dir):
488 ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
492 # Strip maximal common leading white space prefix from text
493 def StripCommon(text):
495 for line in text.split('\n'):
499 white = line[:len(line)-len(line.lstrip())]
504 for i in range(min(len(white), len(ws))+1):
505 if white[0:i] == ws[0:i]:
513 for line in text.split('\n'):
515 if line.startswith(ws):
516 line = line[len(ws):]
517 if line == '' and t == '':
520 while len(t) >= 2 and t[-2:] == '\n\n':
524 # Indent text with indent.
525 def Indent(text, indent):
527 for line in text.split('\n'):
528 t += indent + line + '\n'
531 # Return the first line of l
533 return text.split('\n')[0]
535 _change_prolog = """# Change list.
536 # Lines beginning with # are ignored.
537 # Multi-line values should be indented.
540 #######################################################################
541 # Mercurial helper functions
543 # Get effective change nodes taking into account applied MQ patches
544 def effective_revpair(repo):
546 return cmdutil.revpair(repo, ['qparent'])
548 return cmdutil.revpair(repo, None)
550 # Return list of changed files in repository that match pats.
551 def ChangedFiles(ui, repo, pats, opts):
552 # Find list of files being operated on.
553 matcher = cmdutil.match(repo, pats, opts)
554 node1, node2 = effective_revpair(repo)
555 modified, added, removed = repo.status(node1, node2, matcher)[:3]
556 l = modified + added + removed
560 # Return list of changed files in repository that match pats and still exist.
561 def ChangedExistingFiles(ui, repo, pats, opts):
562 matcher = cmdutil.match(repo, pats, opts)
563 node1, node2 = effective_revpair(repo)
564 modified, added, _ = repo.status(node1, node2, matcher)[:3]
569 # Return list of files claimed by existing CLs
570 def TakenFiles(ui, repo):
571 return Taken(ui, repo).keys()
574 all = LoadAllCL(ui, repo, web=False)
576 for _, cl in all.items():
581 # Return list of changed files that are not claimed by other CLs
582 def DefaultFiles(ui, repo, pats, opts):
583 return Sub(ChangedFiles(ui, repo, pats, opts), TakenFiles(ui, repo))
586 return [l for l in l1 if l not in l2]
593 def Intersect(l1, l2):
594 return [l for l in l1 if l in l2]
596 def getremote(ui, repo, opts):
597 # save $http_proxy; creating the HTTP repo object will
598 # delete it in an attempt to "help"
599 proxy = os.environ.get('http_proxy')
600 source = hg.parseurl(ui.expandpath("default"), None)[0]
602 remoteui = hg.remoteui # hg 1.6
604 remoteui = cmdutil.remoteui
605 other = hg.repository(remoteui(repo, opts), source)
606 if proxy is not None:
607 os.environ['http_proxy'] = proxy
610 def Incoming(ui, repo, opts):
611 _, incoming, _ = findcommonincoming(repo, getremote(ui, repo, opts))
614 def EditCL(ui, repo, cl):
617 s = ui.edit(s, ui.username())
618 clx, line, err = ParseCL(s, cl.name)
620 if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)):
621 return "change list not modified"
624 cl.reviewer = clx.reviewer
628 if promptyesno(ui, "change list should have description\nre-edit (y/n)?"):
633 # For use by submit, etc. (NOT by change)
634 # Get change list number or list of files from command line.
635 # If files are given, make a new change list.
636 def CommandLineCL(ui, repo, pats, opts, defaultcc=None):
637 if len(pats) > 0 and GoodCLName(pats[0]):
639 return None, "cannot specify change number and file names"
640 if opts.get('message'):
641 return None, "cannot use -m with existing CL"
642 cl, err = LoadCL(ui, repo, pats[0], web=True)
648 cl.files = Sub(ChangedFiles(ui, repo, pats, opts), TakenFiles(ui, repo))
650 return None, "no files changed"
651 if opts.get('reviewer'):
652 cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
654 cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
656 cl.cc = Add(cl.cc, defaultcc)
658 if opts.get('message'):
659 cl.desc = opts.get('message')
661 err = EditCL(ui, repo, cl)
666 # reposetup replaces cmdutil.match with this wrapper,
667 # which expands the syntax @clnumber to mean the files
669 original_match = None
670 def ReplacementForCmdutilMatch(repo, pats=[], opts={}, globbed=False, default='relpath'):
675 if p.startswith('@'):
678 if not GoodCLName(clname):
679 raise util.Abort("invalid CL name " + clname)
680 cl, err = LoadCL(repo.ui, repo, clname, web=False)
682 raise util.Abort("loading CL " + clname + ": " + err)
684 raise util.Abort("no files in CL " + clname)
685 files = Add(files, cl.files)
686 pats = Sub(pats, taken) + ['path:'+f for f in files]
687 return original_match(repo, pats=pats, opts=opts, globbed=globbed, default=default)
689 def RelativePath(path, cwd):
691 if path.startswith(cwd) and path[n] == '/':
695 # Check that gofmt run on the list of files does not change them
696 def CheckGofmt(ui, repo, files, just_warn=False):
697 files = [f for f in files if (f.startswith('src/') or f.startswith('test/bench/')) and f.endswith('.go')]
701 files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
702 files = [f for f in files if os.access(f, 0)]
706 cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
709 raise util.Abort("gofmt: " + ExceptionDetail())
710 data = cmd.stdout.read()
711 errors = cmd.stderr.read()
714 ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
717 msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
719 ui.warn("warning: " + msg + "\n")
721 raise util.Abort(msg)
724 #######################################################################
727 # every command must take a ui and and repo as arguments.
728 # opts is a dict where you can find other command line flags
730 # Other parameters are taken in order from items on the command line that
731 # don't start with a dash. If no default value is given in the parameter list,
735 def change(ui, repo, *pats, **opts):
736 """create, edit or delete a change list
738 Create, edit or delete a change list.
739 A change list is a group of files to be reviewed and submitted together,
740 plus a textual description of the change.
741 Change lists are referred to by simple alphanumeric names.
743 Changes must be reviewed before they can be submitted.
745 In the absence of options, the change command opens the
746 change list for editing in the default editor.
748 Deleting a change with the -d or -D flag does not affect
749 the contents of the files listed in that change. To revert
750 the files listed in a change, use
754 before running hg change -d 123456.
758 if len(pats) > 0 and GoodCLName(pats[0]):
761 return "cannot specify CL name and file patterns"
763 cl, err = LoadCL(ui, repo, name, web=True)
766 if not cl.local and (opts["stdin"] or not opts["stdout"]):
767 return "cannot change non-local CL " + name
772 files = ChangedFiles(ui, repo, pats, opts)
773 taken = TakenFiles(ui, repo)
774 files = Sub(files, taken)
776 if opts["delete"] or opts["deletelocal"]:
777 if opts["delete"] and opts["deletelocal"]:
778 return "cannot use -d and -D together"
780 if opts["deletelocal"]:
783 return "cannot use "+flag+" with file patterns"
784 if opts["stdin"] or opts["stdout"]:
785 return "cannot use "+flag+" with -i or -o"
787 return "cannot change non-local CL " + name
790 return "original author must delete CL; hg change -D will remove locally"
791 PostMessage(ui, cl.name, "*** Abandoned ***")
792 EditDesc(cl.name, closed="checked")
798 clx, line, err = ParseCL(s, name)
800 return "error parsing change list: line %d: %s" % (line, err)
801 if clx.desc is not None:
804 if clx.reviewer is not None:
805 cl.reviewer = clx.reviewer
807 if clx.cc is not None:
810 if clx.files is not None:
814 if not opts["stdin"] and not opts["stdout"]:
817 err = EditCL(ui, repo, cl)
822 for d, _ in dirty.items():
826 ui.write(cl.EditorText())
831 ui.write("CL created: " + cl.url + "\n")
834 def code_login(ui, repo, **opts):
835 """log in to code review server
837 Logs in to the code review server, saving a cookie in
838 a file in your home directory.
842 def clpatch(ui, repo, clname, **opts):
843 """import a patch from the code review server
845 Imports a patch from the code review server into the local client.
846 If the local client has already modified any of the files that the
847 patch modifies, this command will refuse to apply the patch.
849 Submitting an imported patch will keep the original author's
850 name as the Author: line but add your own name to a Committer: line.
852 cl, patch, err = DownloadCL(ui, repo, clname)
856 if opts["no_incoming"]:
857 argv += ["--checksync=false"]
861 cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=True)
863 return "hgpatch: " + ExceptionDetail()
865 cmd.stdin.write(patch)
868 out = cmd.stdout.read()
869 if cmd.wait() != 0 and not opts["ignore_hgpatch_failure"]:
870 return "hgpatch failed"
872 cl.files = out.strip().split()
873 files = ChangedFiles(ui, repo, [], opts)
874 extra = Sub(cl.files, files)
876 ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
878 ui.write(cl.PendingText() + "\n")
880 def download(ui, repo, clname, **opts):
881 """download a change from the code review server
883 Download prints a description of the given change list
884 followed by its diff, downloaded from the code review server.
886 cl, patch, err = DownloadCL(ui, repo, clname)
889 ui.write(cl.EditorText() + "\n")
890 ui.write(patch + "\n")
893 def file(ui, repo, clname, pat, *pats, **opts):
894 """assign files to or remove files from a change list
896 Assign files to or (with -d) remove files from a change list.
898 The -d option only removes files from the change list.
899 It does not edit them or remove them from the repository.
901 pats = tuple([pat] + list(pats))
902 if not GoodCLName(clname):
903 return "invalid CL name " + clname
906 cl, err = LoadCL(ui, repo, clname, web=False)
910 return "cannot change non-local CL " + clname
912 files = ChangedFiles(ui, repo, pats, opts)
915 oldfiles = Intersect(files, cl.files)
918 ui.status("# Removing files from CL. To undo:\n")
919 ui.status("# cd %s\n" % (repo.root))
921 ui.status("# hg file %s %s\n" % (cl.name, f))
922 cl.files = Sub(cl.files, oldfiles)
925 ui.status("no such files in CL")
929 return "no such modified files"
931 files = Sub(files, cl.files)
932 taken = Taken(ui, repo)
936 if not warned and not ui.quiet:
937 ui.status("# Taking files from other CLs. To undo:\n")
938 ui.status("# cd %s\n" % (repo.root))
942 ui.status("# hg file %s %s\n" % (ocl.name, f))
944 ocl.files = Sub(ocl.files, files)
946 cl.files = Add(cl.files, files)
948 for d, _ in dirty.items():
952 def gofmt(ui, repo, *pats, **opts):
953 """apply gofmt to modified files
955 Applies gofmt to the modified files in the repository that match
958 files = ChangedExistingFiles(ui, repo, pats, opts)
959 files = [f for f in files if f.endswith(".go")]
961 return "no modified go files"
963 files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
965 cmd = ["gofmt", "-l"]
968 if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0:
969 raise util.Abort("gofmt did not exit cleanly")
970 except error.Abort, e:
973 raise util.Abort("gofmt: " + ExceptionDetail())
976 def mail(ui, repo, *pats, **opts):
977 """mail a change for review
979 Uploads a patch to the code review server and then sends mail
980 to the reviewer and CC list asking for a review.
982 cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
985 cl.Upload(ui, repo, gofmt_just_warn=True)
987 # If no reviewer is listed, assign the review to defaultcc.
988 # This makes sure that it appears in the
989 # codereview.appspot.com/user/defaultcc
990 # page, so that it doesn't get dropped on the floor.
992 return "no reviewers listed in CL"
993 cl.cc = Sub(cl.cc, defaultcc)
994 cl.reviewer = defaultcc
998 def nocommit(ui, repo, *pats, **opts):
999 """(disabled when using this extension)"""
1000 return "The codereview extension is enabled; do not use commit."
1002 def pending(ui, repo, *pats, **opts):
1003 """show pending changes
1005 Lists pending changes followed by a list of unassigned but modified files.
1007 m = LoadAllCL(ui, repo, web=True)
1012 ui.write(cl.PendingText() + "\n")
1014 files = DefaultFiles(ui, repo, [], opts)
1016 s = "Changed files not in any CL:\n"
1018 s += "\t" + f + "\n"
1021 def reposetup(ui, repo):
1022 global original_match
1023 if original_match is None:
1024 original_match = cmdutil.match
1025 cmdutil.match = ReplacementForCmdutilMatch
1026 RietveldSetup(ui, repo)
1028 def CheckContributor(ui, repo, user=None):
1030 user = ui.config("ui", "username")
1032 raise util.Abort("[ui] username is not configured in .hgrc")
1033 _, userline = FindContributor(ui, repo, user, warn=False)
1035 raise util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
1038 def FindContributor(ui, repo, user, warn=True):
1039 m = re.match(r".*<(.*)>", user)
1041 user = m.group(1).lower()
1043 if user not in contributors:
1045 ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1048 user, email = contributors[user]
1049 return email, "%s <%s>" % (user, email)
1051 def submit(ui, repo, *pats, **opts):
1052 """submit change to remote repository
1054 Submits change to remote repository.
1055 Bails out if the local repository is not in sync with the remote one.
1057 repo.ui.quiet = True
1058 if not opts["no_incoming"] and Incoming(ui, repo, opts):
1059 return "local repository out of date; must sync before submit"
1061 cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1067 user = cl.copied_from
1068 userline = CheckContributor(ui, repo, user)
1072 about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n"
1074 tbr = SplitCommaSpace(opts.get('tbr'))
1075 cl.reviewer = Add(cl.reviewer, tbr)
1076 about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1078 about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1081 return "no reviewers listed in CL"
1084 return "cannot submit non-local CL"
1086 # upload, to sync current patch and also get change number if CL is new.
1087 if not cl.copied_from:
1088 cl.Upload(ui, repo, gofmt_just_warn=True)
1090 # check gofmt for real; allowed upload to warn in order to save CL.
1092 CheckGofmt(ui, repo, cl.files)
1094 about += "%s%s\n" % (server_url_base, cl.name)
1097 about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1099 if not cl.mailed and not cl.copied_from: # in case this is TBR
1102 # submit changes locally
1103 date = opts.get('date')
1105 opts['date'] = util.parsedate(date)
1106 opts['message'] = cl.desc.rstrip() + "\n\n" + about
1109 print "NOT SUBMITTING:"
1110 print "User: ", userline
1112 print Indent(opts['message'], "\t")
1114 print Indent('\n'.join(cl.files), "\t")
1115 return "dry run; not submitted"
1117 m = match.exact(repo.root, repo.getcwd(), cl.files)
1118 node = repo.commit(opts['message'], userline, opts.get('date'), m)
1120 return "nothing changed"
1122 # push to remote; if it fails for any reason, roll back
1124 log = repo.changelog
1126 parents = log.parentrevs(rev)
1127 if (rev-1 not in parents and
1128 (parents == (nullrev, nullrev) or
1129 len(log.heads(log.node(parents[0]))) > 1 and
1130 (parents[1] == nullrev or len(log.heads(log.node(parents[1]))) > 1))):
1132 raise util.Abort("local repository out of date; must sync before submit")
1134 # push changes to remote.
1135 # if it works, we're committed.
1137 other = getremote(ui, repo, opts)
1138 r = repo.push(other, False, None)
1140 raise util.Abort("local repository out of date; must sync before submit")
1145 # we're committed. upload final patch, close review, add commit message
1146 changeURL = short(node)
1148 m = re.match("^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/", url)
1150 changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(2), changeURL)
1152 print >>sys.stderr, "URL: ", url
1153 pmsg = "*** Submitted as " + changeURL + " ***\n\n" + opts['message']
1155 # When posting, move reviewers to CC line,
1156 # so that the issue stops showing up in their "My Issues" page.
1157 PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
1159 if not cl.copied_from:
1160 EditDesc(cl.name, closed="checked")
1163 def sync(ui, repo, **opts):
1164 """synchronize with remote repository
1166 Incorporates recent changes from the remote repository
1167 into the local repository.
1169 if not opts["local"]:
1170 ui.status = sync_note
1172 other = getremote(ui, repo, opts)
1173 modheads = repo.pull(other)
1174 err = commands.postincoming(ui, repo, modheads, True, "tip")
1177 commands.update(ui, repo)
1178 sync_changes(ui, repo)
1181 # we run sync (pull -u) in verbose mode to get the
1182 # list of files being updated, but that drags along
1183 # a bunch of messages we don't care about.
1185 if msg == 'resolving manifests\n':
1187 if msg == 'searching for changes\n':
1189 if msg == "couldn't find merge tool hgmerge\n":
1191 sys.stdout.write(msg)
1193 def sync_changes(ui, repo):
1194 # Look through recent change log descriptions to find
1195 # potential references to http://.*/our-CL-number.
1196 # Double-check them by looking at the Rietveld log.
1198 desc = repo[rev].description().strip()
1199 for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc):
1200 if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
1201 ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
1202 cl, err = LoadCL(ui, repo, clname, web=False)
1204 ui.warn("loading CL %s: %s\n" % (clname, err))
1206 if not cl.copied_from:
1207 EditDesc(cl.name, closed="checked")
1210 if hgversion < '1.4':
1211 get = util.cachefunc(lambda r: repo[r].changeset())
1212 changeiter, matchfn = cmdutil.walkchangerevs(ui, repo, [], get, {'rev': None})
1214 for st, rev, fns in changeiter:
1222 matchfn = cmdutil.match(repo, [], {'rev': None})
1225 for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev': None}, prep):
1228 # Remove files that are not modified from the CLs in which they appear.
1229 all = LoadAllCL(ui, repo, web=False)
1230 changed = ChangedFiles(ui, repo, [], {})
1231 for _, cl in all.items():
1232 extra = Sub(cl.files, changed)
1234 ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
1236 ui.warn("\t%s\n" % (f,))
1237 cl.files = Sub(cl.files, extra)
1240 ui.warn("CL %s has no files; suggest hg change -d %s\n" % (cl.name, cl.name))
1244 if "^commit|ci" in commands.table:
1245 commands.table["^commit|ci"] = (nocommit, [], "")
1247 def upload(ui, repo, name, **opts):
1248 """upload diffs to the code review server
1250 Uploads the current modifications for a given change to the server.
1252 repo.ui.quiet = True
1253 cl, err = LoadCL(ui, repo, name, web=True)
1257 return "cannot upload non-local change"
1259 print "%s%s\n" % (server_url_base, cl.name)
1263 ('r', 'reviewer', '', 'add reviewer'),
1264 ('', 'cc', '', 'add cc'),
1265 ('', 'tbr', '', 'add future reviewer'),
1266 ('m', 'message', '', 'change description (for new change)'),
1270 # The ^ means to show this command in the help text that
1271 # is printed when running hg with no arguments.
1275 ('d', 'delete', None, 'delete existing change list'),
1276 ('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
1277 ('i', 'stdin', None, 'read change list from standard input'),
1278 ('o', 'stdout', None, 'print change list to standard output'),
1280 "[-d | -D] [-i] [-o] change# or FILE ..."
1285 ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
1286 ('', 'no_incoming', None, 'disable check for incoming changes'),
1287 ('', 'fuzzy', None, 'attempt to adjust patch line numbers'),
1291 # Would prefer to call this codereview-login, but then
1292 # hg help codereview prints the help for this command
1293 # instead of the help for the extension.
1312 ('d', 'delete', None, 'delete files from change list (but not repository)'),
1314 "[-d] change# FILE ..."
1319 ('l', 'list', None, 'list files that would change, but do not edit them'),
1331 ] + commands.walkopts,
1332 "[-r reviewer] [--cc cc] [change# | file ...]"
1337 ('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
1338 ('n', 'dryrun', None, 'make change only locally (for testing)'),
1339 ] + commands.walkopts + commands.commitopts + commands.commitopts2,
1340 "[-r reviewer] [--cc cc] [change# | file ...]"
1345 ('', 'local', None, 'do not pull changes from remote repository')
1357 #######################################################################
1358 # Wrappers around upload.py for interacting with Rietveld
1361 class FormParser(HTMLParser):
1366 HTMLParser.__init__(self)
1367 def handle_starttag(self, tag, attrs):
1377 self.map[key] = value
1378 if tag == "textarea":
1386 def handle_endtag(self, tag):
1387 if tag == "textarea" and self.curtag is not None:
1388 self.map[self.curtag] = self.curdata
1391 def handle_charref(self, name):
1392 self.handle_data(unichr(int(name)))
1393 def handle_entityref(self, name):
1394 import htmlentitydefs
1395 if name in htmlentitydefs.entitydefs:
1396 self.handle_data(htmlentitydefs.entitydefs[name])
1398 self.handle_data("&" + name + ";")
1399 def handle_data(self, data):
1400 if self.curdata is not None:
1401 self.curdata += data.decode("utf-8").encode("utf-8")
1404 def XMLGet(ui, path):
1406 data = MySend(path, force_auth=False);
1408 ui.warn("XMLGet %s: %s\n" % (path, ExceptionDetail()))
1412 def IsRietveldSubmitted(ui, clname, hex):
1413 feed = XMLGet(ui, "/rss/issue/" + clname)
1416 for sum in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}summary"):
1417 text = sum.findtext("", None).strip()
1418 m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text)
1419 if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
1423 def DownloadCL(ui, repo, clname):
1424 cl, err = LoadCL(ui, repo, clname)
1426 return None, None, "error loading CL %s: %s" % (clname, ExceptionDetail())
1428 # Grab RSS feed to learn about CL
1429 feed = XMLGet(ui, "/rss/issue/" + clname)
1431 return None, None, "cannot download CL"
1433 # Find most recent diff
1435 prefix = 'http://' + server + '/'
1436 for link in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}link"):
1437 if link.get('rel') != 'alternate':
1439 text = link.get('href')
1440 if not text.startswith(prefix) or not text.endswith('.diff'):
1442 diff = text[len(prefix)-1:]
1444 return None, None, "CL has no diff"
1445 diffdata = MySend(diff, force_auth=False)
1447 # Find author - first entry will be author who created CL.
1449 for author in feed.findall("{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}author/{http://www.w3.org/2005/Atom}name"):
1450 nick = author.findtext("", None).strip()
1453 return None, None, "CL has no author"
1455 # The author is just a nickname: get the real email address.
1457 # want URL-encoded nick, but without a=, and rietveld rejects + for %20.
1458 url = "/user_popup/" + urllib.urlencode({"a": nick})[2:].replace("+", "%20")
1459 data = MySend(url, force_auth=False)
1461 ui.warn("error looking up %s: %s\n" % (nick, ExceptionDetail()))
1462 cl.copied_from = nick+"@needtofix"
1463 return cl, diffdata, ""
1464 match = re.match(r"<b>(.*) \((.*)\)</b>", data)
1466 return None, None, "error looking up %s: cannot parse result %s" % (nick, repr(data))
1467 if match.group(1) != nick and match.group(2) != nick:
1468 return None, None, "error looking up %s: got info for %s, %s" % (nick, match.group(1), match.group(2))
1469 email = match.group(1)
1471 # Print warning if email is not in CONTRIBUTORS file.
1472 FindContributor(ui, repo, email)
1473 cl.copied_from = email
1475 return cl, diffdata, ""
1477 def MySend(request_path, payload=None,
1478 content_type="application/octet-stream",
1479 timeout=None, force_auth=True,
1481 """Run MySend1 maybe twice, because Rietveld is unreliable."""
1483 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
1484 except Exception, e:
1485 if type(e) == urllib2.HTTPError and e.code == 403: # forbidden, it happens
1487 print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
1489 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
1492 # Like upload.py Send but only authenticates when the
1493 # redirect is to www.google.com/accounts. This keeps
1494 # unnecessary redirects from happening during testing.
1495 def MySend1(request_path, payload=None,
1496 content_type="application/octet-stream",
1497 timeout=None, force_auth=True,
1499 """Sends an RPC and returns the response.
1502 request_path: The path to send the request to, eg /api/appversion/create.
1503 payload: The body of the request, or None to send an empty request.
1504 content_type: The Content-Type header to use.
1505 timeout: timeout in seconds; default None i.e. no timeout.
1506 (Note: for large requests on OS X, the timeout doesn't work right.)
1507 kwargs: Any keyword arguments are converted into query string parameters.
1510 The response body, as a string.
1512 # TODO: Don't require authentication. Let the server say
1513 # whether it is necessary.
1516 rpc = GetRpcServer(upload_options)
1518 if not self.authenticated and force_auth:
1519 self._Authenticate()
1520 if request_path is None:
1523 old_timeout = socket.getdefaulttimeout()
1524 socket.setdefaulttimeout(timeout)
1530 url = "http://%s%s" % (self.host, request_path)
1532 url += "?" + urllib.urlencode(args)
1533 req = self._CreateRequest(url=url, data=payload)
1534 req.add_header("Content-Type", content_type)
1536 f = self.opener.open(req)
1539 # Translate \r\n into \n, because Rietveld doesn't.
1540 response = response.replace('\r\n', '\n')
1542 except urllib2.HTTPError, e:
1546 self._Authenticate()
1548 loc = e.info()["location"]
1549 if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
1551 self._Authenticate()
1555 socket.setdefaulttimeout(old_timeout)
1561 for k,v in f.map.items():
1562 f.map[k] = v.replace("\r\n", "\n");
1565 # Fetch the settings for the CL, like reviewer and CC list, by
1566 # scraping the Rietveld editing forms.
1567 def GetSettings(issue):
1568 # The /issue/edit page has everything but only the
1569 # CL owner is allowed to fetch it (and submit it).
1572 f = GetForm("/" + issue + "/edit")
1575 if not f or 'reviewers' not in f:
1576 # Maybe we're not the CL owner. Fall back to the
1577 # /publish page, which has the reviewer and CC lists,
1578 # and then fetch the description separately.
1579 f = GetForm("/" + issue + "/publish")
1580 f['description'] = MySend("/"+issue+"/description", force_auth=False)
1583 def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=None):
1584 form_fields = GetForm("/" + issue + "/edit")
1585 if subject is not None:
1586 form_fields['subject'] = subject
1587 if desc is not None:
1588 form_fields['description'] = desc
1589 if reviewers is not None:
1590 form_fields['reviewers'] = reviewers
1592 form_fields['cc'] = cc
1593 if closed is not None:
1594 form_fields['closed'] = closed
1595 ctype, body = EncodeMultipartFormData(form_fields.items(), [])
1596 response = MySend("/" + issue + "/edit", body, content_type=ctype)
1598 print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
1601 def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
1602 form_fields = GetForm("/" + issue + "/publish")
1603 if reviewers is not None:
1604 form_fields['reviewers'] = reviewers
1606 form_fields['cc'] = cc
1608 form_fields['send_mail'] = "checked"
1610 del form_fields['send_mail']
1611 if subject is not None:
1612 form_fields['subject'] = subject
1613 form_fields['message'] = message
1615 form_fields['message_only'] = '1' # Don't include draft comments
1616 if reviewers is not None or cc is not None:
1617 form_fields['message_only'] = '' # Must set '' in order to override cc/reviewer
1618 ctype = "applications/x-www-form-urlencoded"
1619 body = urllib.urlencode(form_fields)
1620 response = MySend("/" + issue + "/publish", body, content_type=ctype)
1628 def RietveldSetup(ui, repo):
1629 global defaultcc, upload_options, rpc, server, server_url_base, force_google_account, verbosity, contributors
1631 # Read repository-specific options from lib/codereview/codereview.cfg
1633 f = open(repo.root + '/lib/codereview/codereview.cfg')
1635 if line.startswith('defaultcc: '):
1636 defaultcc = SplitCommaSpace(line[10:])
1638 # If there are no options, chances are good this is not
1639 # a code review repository; stop now before we foul
1640 # things up even worse. Might also be that repo doesn't
1641 # even have a root. See issue 959.
1645 f = open(repo.root + '/CONTRIBUTORS', 'r')
1647 raise util.Abort("cannot open %s: %s" % (repo.root+'/CONTRIBUTORS', ExceptionDetail()))
1649 # CONTRIBUTORS is a list of lines like:
1651 # Person <email> <alt-email>
1652 # The first email address is the one used in commit logs.
1653 if line.startswith('#'):
1655 m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line)
1658 email = m.group(2)[1:-1]
1659 contributors[email.lower()] = (name, email)
1660 for extra in m.group(3).split():
1661 contributors[extra[1:-1].lower()] = (name, email)
1664 # TODO(rsc): If the repository config has no codereview section,
1665 # do not enable the extension. This allows users to
1666 # put the extension in their global .hgrc but only
1667 # enable it for some repositories.
1668 # if not ui.has_section("codereview"):
1676 x = ui.config("codereview", "server")
1680 # TODO(rsc): Take from ui.username?
1682 x = ui.config("codereview", "email")
1686 server_url_base = "http://" + server + "/"
1688 testing = ui.config("codereview", "testing")
1689 force_google_account = ui.configbool("codereview", "force_google_account", False)
1691 upload_options = opt()
1692 upload_options.email = email
1693 upload_options.host = None
1694 upload_options.verbose = 0
1695 upload_options.description = None
1696 upload_options.description_file = None
1697 upload_options.reviewers = None
1698 upload_options.cc = None
1699 upload_options.message = None
1700 upload_options.issue = None
1701 upload_options.download_base = False
1702 upload_options.revision = None
1703 upload_options.send_mail = False
1704 upload_options.vcs = None
1705 upload_options.server = server
1706 upload_options.save_cookies = True
1709 upload_options.save_cookies = False
1710 upload_options.email = "test@example.com"
1714 #######################################################################
1715 # We keep a full copy of upload.py here to avoid import path hell.
1716 # It would be nice if hg added the hg repository root
1717 # to the default PYTHONPATH.
1719 # Edit .+2,<hget http://codereview.appspot.com/static/upload.py
1721 #!/usr/bin/env python
1723 # Copyright 2007 Google Inc.
1725 # Licensed under the Apache License, Version 2.0 (the "License");
1726 # you may not use this file except in compliance with the License.
1727 # You may obtain a copy of the License at
1729 # http://www.apache.org/licenses/LICENSE-2.0
1731 # Unless required by applicable law or agreed to in writing, software
1732 # distributed under the License is distributed on an "AS IS" BASIS,
1733 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1734 # See the License for the specific language governing permissions and
1735 # limitations under the License.
1737 """Tool for uploading diffs from a version control system to the codereview app.
1739 Usage summary: upload.py [options] [-- diff_options]
1741 Diff options are passed to the diff command of the underlying system.
1743 Supported version control systems:
1748 It is important for Git/Mercurial users to specify a tree/node/branch to diff
1749 against by using the '--rev' option.
1751 # This code is derived from appcfg.py in the App Engine SDK (open source),
1752 # and from ASPN recipe #146306.
1768 # The md5 module was deprecated in Python 2.5.
1770 from hashlib import md5
1779 # The logging verbosity:
1781 # 1: Status messages.
1786 # Max size of patch or base file.
1787 MAX_UPLOAD_SIZE = 900 * 1024
1789 # Constants for version control names. Used by GuessVCSName.
1791 VCS_MERCURIAL = "Mercurial"
1792 VCS_SUBVERSION = "Subversion"
1793 VCS_UNKNOWN = "Unknown"
1795 # whitelist for non-binary filetypes which do not start with "text/"
1796 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
1797 TEXT_MIMETYPES = ['application/javascript', 'application/x-javascript',
1798 'application/x-freemind']
1800 VCS_ABBREVIATIONS = {
1801 VCS_MERCURIAL.lower(): VCS_MERCURIAL,
1802 "hg": VCS_MERCURIAL,
1803 VCS_SUBVERSION.lower(): VCS_SUBVERSION,
1804 "svn": VCS_SUBVERSION,
1805 VCS_GIT.lower(): VCS_GIT,
1809 def GetEmail(prompt):
1810 """Prompts the user for their email address and returns it.
1812 The last used email address is saved to a file and offered up as a suggestion
1813 to the user. If the user presses enter without typing in anything the last
1814 used email address is used. If the user enters a new address, it is saved
1815 for next time we prompt.
1818 last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
1820 if os.path.exists(last_email_file_name):
1822 last_email_file = open(last_email_file_name, "r")
1823 last_email = last_email_file.readline().strip("\n")
1824 last_email_file.close()
1825 prompt += " [%s]" % last_email
1828 email = raw_input(prompt + ": ").strip()
1831 last_email_file = open(last_email_file_name, "w")
1832 last_email_file.write(email)
1833 last_email_file.close()
1841 def StatusUpdate(msg):
1842 """Print a status message to stdout.
1844 If 'verbosity' is greater than 0, print the message.
1847 msg: The string to print.
1854 """Print an error message to stderr and exit."""
1855 print >>sys.stderr, msg
1859 class ClientLoginError(urllib2.HTTPError):
1860 """Raised to indicate there was an error authenticating with ClientLogin."""
1862 def __init__(self, url, code, msg, headers, args):
1863 urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
1865 self.reason = args["Error"]
1868 class AbstractRpcServer(object):
1869 """Provides a common interface for a simple RPC server."""
1871 def __init__(self, host, auth_function, host_override=None, extra_headers={},
1872 save_cookies=False):
1873 """Creates a new HttpRpcServer.
1876 host: The host to send requests to.
1877 auth_function: A function that takes no arguments and returns an
1878 (email, password) tuple when called. Will be called if authentication
1880 host_override: The host header to send to the server (defaults to host).
1881 extra_headers: A dict of extra headers to append to every request.
1882 save_cookies: If True, save the authentication cookies to local disk.
1883 If False, use an in-memory cookiejar instead. Subclasses must
1884 implement this functionality. Defaults to False.
1887 self.host_override = host_override
1888 self.auth_function = auth_function
1889 self.authenticated = False
1890 self.extra_headers = extra_headers
1891 self.save_cookies = save_cookies
1892 self.opener = self._GetOpener()
1893 if self.host_override:
1894 logging.info("Server: %s; Host: %s", self.host, self.host_override)
1896 logging.info("Server: %s", self.host)
1898 def _GetOpener(self):
1899 """Returns an OpenerDirector for making HTTP requests.
1902 A urllib2.OpenerDirector object.
1904 raise NotImplementedError()
1906 def _CreateRequest(self, url, data=None):
1907 """Creates a new urllib request."""
1908 logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
1909 req = urllib2.Request(url, data=data)
1910 if self.host_override:
1911 req.add_header("Host", self.host_override)
1912 for key, value in self.extra_headers.iteritems():
1913 req.add_header(key, value)
1916 def _GetAuthToken(self, email, password):
1917 """Uses ClientLogin to authenticate the user, returning an auth token.
1920 email: The user's email address
1921 password: The user's password
1924 ClientLoginError: If there was an error authenticating with ClientLogin.
1925 HTTPError: If there was some other form of HTTP error.
1928 The authentication token returned by ClientLogin.
1930 account_type = "GOOGLE"
1931 if self.host.endswith(".google.com") and not force_google_account:
1932 # Needed for use inside Google.
1933 account_type = "HOSTED"
1934 req = self._CreateRequest(
1935 url="https://www.google.com/accounts/ClientLogin",
1936 data=urllib.urlencode({
1940 "source": "rietveld-codereview-upload",
1941 "accountType": account_type,
1945 response = self.opener.open(req)
1946 response_body = response.read()
1947 response_dict = dict(x.split("=")
1948 for x in response_body.split("\n") if x)
1949 return response_dict["Auth"]
1950 except urllib2.HTTPError, e:
1953 response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
1954 raise ClientLoginError(req.get_full_url(), e.code, e.msg,
1955 e.headers, response_dict)
1959 def _GetAuthCookie(self, auth_token):
1960 """Fetches authentication cookies for an authentication token.
1963 auth_token: The authentication token returned by ClientLogin.
1966 HTTPError: If there was an error fetching the authentication cookies.
1968 # This is a dummy value to allow us to identify when we're successful.
1969 continue_location = "http://localhost/"
1970 args = {"continue": continue_location, "auth": auth_token}
1971 req = self._CreateRequest("http://%s/_ah/login?%s" %
1972 (self.host, urllib.urlencode(args)))
1974 response = self.opener.open(req)
1975 except urllib2.HTTPError, e:
1977 if (response.code != 302 or
1978 response.info()["location"] != continue_location):
1979 raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
1980 response.headers, response.fp)
1981 self.authenticated = True
1983 def _Authenticate(self):
1984 """Authenticates the user.
1986 The authentication process works as follows:
1987 1) We get a username and password from the user
1988 2) We use ClientLogin to obtain an AUTH token for the user
1989 (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
1990 3) We pass the auth token to /_ah/login on the server to obtain an
1991 authentication cookie. If login was successful, it tries to redirect
1992 us to the URL we provided.
1994 If we attempt to access the upload API without first obtaining an
1995 authentication cookie, it returns a 401 response (or a 302) and
1996 directs us to authenticate ourselves with ClientLogin.
1999 credentials = self.auth_function()
2001 auth_token = self._GetAuthToken(credentials[0], credentials[1])
2002 except ClientLoginError, e:
2003 if e.reason == "BadAuthentication":
2004 print >>sys.stderr, "Invalid username or password."
2006 if e.reason == "CaptchaRequired":
2007 print >>sys.stderr, (
2009 "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
2010 "and verify you are a human. Then try again.")
2012 if e.reason == "NotVerified":
2013 print >>sys.stderr, "Account not verified."
2015 if e.reason == "TermsNotAgreed":
2016 print >>sys.stderr, "User has not agreed to TOS."
2018 if e.reason == "AccountDeleted":
2019 print >>sys.stderr, "The user account has been deleted."
2021 if e.reason == "AccountDisabled":
2022 print >>sys.stderr, "The user account has been disabled."
2024 if e.reason == "ServiceDisabled":
2025 print >>sys.stderr, ("The user's access to the service has been "
2028 if e.reason == "ServiceUnavailable":
2029 print >>sys.stderr, "The service is not available; try again later."
2032 self._GetAuthCookie(auth_token)
2035 def Send(self, request_path, payload=None,
2036 content_type="application/octet-stream",
2039 """Sends an RPC and returns the response.
2042 request_path: The path to send the request to, eg /api/appversion/create.
2043 payload: The body of the request, or None to send an empty request.
2044 content_type: The Content-Type header to use.
2045 timeout: timeout in seconds; default None i.e. no timeout.
2046 (Note: for large requests on OS X, the timeout doesn't work right.)
2047 kwargs: Any keyword arguments are converted into query string parameters.
2050 The response body, as a string.
2052 # TODO: Don't require authentication. Let the server say
2053 # whether it is necessary.
2054 if not self.authenticated:
2055 self._Authenticate()
2057 old_timeout = socket.getdefaulttimeout()
2058 socket.setdefaulttimeout(timeout)
2064 url = "http://%s%s" % (self.host, request_path)
2066 url += "?" + urllib.urlencode(args)
2067 req = self._CreateRequest(url=url, data=payload)
2068 req.add_header("Content-Type", content_type)
2070 f = self.opener.open(req)
2074 except urllib2.HTTPError, e:
2077 elif e.code == 401 or e.code == 302:
2078 self._Authenticate()
2082 socket.setdefaulttimeout(old_timeout)
2085 class HttpRpcServer(AbstractRpcServer):
2086 """Provides a simplified RPC-style interface for HTTP requests."""
2088 def _Authenticate(self):
2089 """Save the cookie jar after authentication."""
2090 super(HttpRpcServer, self)._Authenticate()
2091 if self.save_cookies:
2092 StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
2093 self.cookie_jar.save()
2095 def _GetOpener(self):
2096 """Returns an OpenerDirector that supports cookies and ignores redirects.
2099 A urllib2.OpenerDirector object.
2101 opener = urllib2.OpenerDirector()
2102 opener.add_handler(urllib2.ProxyHandler())
2103 opener.add_handler(urllib2.UnknownHandler())
2104 opener.add_handler(urllib2.HTTPHandler())
2105 opener.add_handler(urllib2.HTTPDefaultErrorHandler())
2106 opener.add_handler(urllib2.HTTPSHandler())
2107 opener.add_handler(urllib2.HTTPErrorProcessor())
2108 if self.save_cookies:
2109 self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
2110 self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
2111 if os.path.exists(self.cookie_file):
2113 self.cookie_jar.load()
2114 self.authenticated = True
2115 StatusUpdate("Loaded authentication cookies from %s" %
2117 except (cookielib.LoadError, IOError):
2118 # Failed to load cookies - just ignore them.
2121 # Create an empty cookie file with mode 600
2122 fd = os.open(self.cookie_file, os.O_CREAT, 0600)
2124 # Always chmod the cookie file
2125 os.chmod(self.cookie_file, 0600)
2127 # Don't save cookies across runs of update.py.
2128 self.cookie_jar = cookielib.CookieJar()
2129 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
2133 parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
2134 parser.add_option("-y", "--assume_yes", action="store_true",
2135 dest="assume_yes", default=False,
2136 help="Assume that the answer to yes/no questions is 'yes'.")
2138 group = parser.add_option_group("Logging options")
2139 group.add_option("-q", "--quiet", action="store_const", const=0,
2140 dest="verbose", help="Print errors only.")
2141 group.add_option("-v", "--verbose", action="store_const", const=2,
2142 dest="verbose", default=1,
2143 help="Print info level logs (default).")
2144 group.add_option("--noisy", action="store_const", const=3,
2145 dest="verbose", help="Print all logs.")
2147 group = parser.add_option_group("Review server options")
2148 group.add_option("-s", "--server", action="store", dest="server",
2149 default="codereview.appspot.com",
2151 help=("The server to upload to. The format is host[:port]. "
2152 "Defaults to '%default'."))
2153 group.add_option("-e", "--email", action="store", dest="email",
2154 metavar="EMAIL", default=None,
2155 help="The username to use. Will prompt if omitted.")
2156 group.add_option("-H", "--host", action="store", dest="host",
2157 metavar="HOST", default=None,
2158 help="Overrides the Host header sent with all RPCs.")
2159 group.add_option("--no_cookies", action="store_false",
2160 dest="save_cookies", default=True,
2161 help="Do not save authentication cookies to local disk.")
2163 group = parser.add_option_group("Issue options")
2164 group.add_option("-d", "--description", action="store", dest="description",
2165 metavar="DESCRIPTION", default=None,
2166 help="Optional description when creating an issue.")
2167 group.add_option("-f", "--description_file", action="store",
2168 dest="description_file", metavar="DESCRIPTION_FILE",
2170 help="Optional path of a file that contains "
2171 "the description when creating an issue.")
2172 group.add_option("-r", "--reviewers", action="store", dest="reviewers",
2173 metavar="REVIEWERS", default=None,
2174 help="Add reviewers (comma separated email addresses).")
2175 group.add_option("--cc", action="store", dest="cc",
2176 metavar="CC", default=None,
2177 help="Add CC (comma separated email addresses).")
2178 group.add_option("--private", action="store_true", dest="private",
2180 help="Make the issue restricted to reviewers and those CCed")
2182 group = parser.add_option_group("Patch options")
2183 group.add_option("-m", "--message", action="store", dest="message",
2184 metavar="MESSAGE", default=None,
2185 help="A message to identify the patch. "
2186 "Will prompt if omitted.")
2187 group.add_option("-i", "--issue", type="int", action="store",
2188 metavar="ISSUE", default=None,
2189 help="Issue number to which to add. Defaults to new issue.")
2190 group.add_option("--download_base", action="store_true",
2191 dest="download_base", default=False,
2192 help="Base files will be downloaded by the server "
2193 "(side-by-side diffs may not work on files with CRs).")
2194 group.add_option("--rev", action="store", dest="revision",
2195 metavar="REV", default=None,
2196 help="Branch/tree/revision to diff against (used by DVCS).")
2197 group.add_option("--send_mail", action="store_true",
2198 dest="send_mail", default=False,
2199 help="Send notification email to reviewers.")
2200 group.add_option("--vcs", action="store", dest="vcs",
2201 metavar="VCS", default=None,
2202 help=("Version control system (optional, usually upload.py "
2203 "already guesses the right VCS)."))
2206 def GetRpcServer(options):
2207 """Returns an instance of an AbstractRpcServer.
2210 A new AbstractRpcServer, on which RPC calls can be made.
2213 rpc_server_class = HttpRpcServer
2215 def GetUserCredentials():
2216 """Prompts the user for a username and password."""
2217 email = options.email
2219 email = GetEmail("Email (login for uploading to %s)" % options.server)
2220 password = getpass.getpass("Password for %s: " % email)
2221 return (email, password)
2223 # If this is the dev_appserver, use fake authentication.
2224 host = (options.host or options.server).lower()
2225 if host == "localhost" or host.startswith("localhost:"):
2226 email = options.email
2228 email = "test@example.com"
2229 logging.info("Using debug user %s. Override with --email" % email)
2230 server = rpc_server_class(
2232 lambda: (email, "password"),
2233 host_override=options.host,
2234 extra_headers={"Cookie":
2235 'dev_appserver_login="%s:False"' % email},
2236 save_cookies=options.save_cookies)
2237 # Don't try to talk to ClientLogin.
2238 server.authenticated = True
2241 return rpc_server_class(options.server, GetUserCredentials,
2242 host_override=options.host,
2243 save_cookies=options.save_cookies)
2246 def EncodeMultipartFormData(fields, files):
2247 """Encode form fields for multipart/form-data.
2250 fields: A sequence of (name, value) elements for regular form fields.
2251 files: A sequence of (name, filename, value) elements for data to be
2254 (content_type, body) ready for httplib.HTTP instance.
2257 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
2259 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
2262 for (key, value) in fields:
2263 lines.append('--' + BOUNDARY)
2264 lines.append('Content-Disposition: form-data; name="%s"' % key)
2266 if type(value) == unicode:
2267 value = value.encode("utf-8")
2269 for (key, filename, value) in files:
2270 if type(filename) == unicode:
2271 filename = filename.encode("utf-8")
2272 if type(value) == unicode:
2273 value = value.encode("utf-8")
2274 lines.append('--' + BOUNDARY)
2275 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
2277 lines.append('Content-Type: %s' % GetContentType(filename))
2280 lines.append('--' + BOUNDARY + '--')
2282 body = CRLF.join(lines)
2283 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
2284 return content_type, body
2287 def GetContentType(filename):
2288 """Helper to guess the content-type from the filename."""
2289 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
2292 # Use a shell for subcommands on Windows to get a PATH search.
2293 use_shell = sys.platform.startswith("win")
2295 def RunShellWithReturnCode(command, print_output=False,
2296 universal_newlines=True,
2298 """Executes a command and returns the output from stdout and the return code.
2301 command: Command to execute.
2302 print_output: If True, the output is printed to stdout.
2303 If False, both stdout and stderr are ignored.
2304 universal_newlines: Use universal_newlines flag (default: True).
2307 Tuple (output, return code)
2309 logging.info("Running %s", command)
2310 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2311 shell=use_shell, universal_newlines=universal_newlines,
2316 line = p.stdout.readline()
2319 print line.strip("\n")
2320 output_array.append(line)
2321 output = "".join(output_array)
2323 output = p.stdout.read()
2325 errout = p.stderr.read()
2326 if print_output and errout:
2327 print >>sys.stderr, errout
2330 return output, p.returncode
2333 def RunShell(command, silent_ok=False, universal_newlines=True,
2334 print_output=False, env=os.environ):
2335 data, retcode = RunShellWithReturnCode(command, print_output,
2336 universal_newlines, env)
2338 ErrorExit("Got error status from %s:\n%s" % (command, data))
2339 if not silent_ok and not data:
2340 ErrorExit("No output from %s" % command)
2344 class VersionControlSystem(object):
2345 """Abstract base class providing an interface to the VCS."""
2347 def __init__(self, options):
2351 options: Command line options.
2353 self.options = options
2355 def GenerateDiff(self, args):
2356 """Return the current diff as a string.
2359 args: Extra arguments to pass to the diff command.
2361 raise NotImplementedError(
2362 "abstract method -- subclass %s must override" % self.__class__)
2364 def GetUnknownFiles(self):
2365 """Return a list of files unknown to the VCS."""
2366 raise NotImplementedError(
2367 "abstract method -- subclass %s must override" % self.__class__)
2369 def CheckForUnknownFiles(self):
2370 """Show an "are you sure?" prompt if there are unknown files."""
2371 unknown_files = self.GetUnknownFiles()
2373 print "The following files are not added to version control:"
2374 for line in unknown_files:
2376 prompt = "Are you sure to continue?(y/N) "
2377 answer = raw_input(prompt).strip()
2379 ErrorExit("User aborted")
2381 def GetBaseFile(self, filename):
2382 """Get the content of the upstream version of a file.
2385 A tuple (base_content, new_content, is_binary, status)
2386 base_content: The contents of the base file.
2387 new_content: For text files, this is empty. For binary files, this is
2388 the contents of the new file, since the diff output won't contain
2389 information to reconstruct the current file.
2390 is_binary: True iff the file is binary.
2391 status: The status of the file.
2394 raise NotImplementedError(
2395 "abstract method -- subclass %s must override" % self.__class__)
2398 def GetBaseFiles(self, diff):
2399 """Helper that calls GetBase file for each file in the patch.
2402 A dictionary that maps from filename to GetBaseFile's tuple. Filenames
2403 are retrieved based on lines that start with "Index:" or
2404 "Property changes on:".
2407 for line in diff.splitlines(True):
2408 if line.startswith('Index:') or line.startswith('Property changes on:'):
2409 unused, filename = line.split(':', 1)
2410 # On Windows if a file has property changes its filename uses '\'
2412 filename = filename.strip().replace('\\', '/')
2413 files[filename] = self.GetBaseFile(filename)
2417 def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
2419 """Uploads the base files (and if necessary, the current ones as well)."""
2421 def UploadFile(filename, file_id, content, is_binary, status, is_base):
2422 """Uploads a file to the server."""
2423 file_too_large = False
2428 if len(content) > MAX_UPLOAD_SIZE:
2429 print ("Not uploading the %s file for %s because it's too large." %
2431 file_too_large = True
2433 checksum = md5(content).hexdigest()
2434 if options.verbose > 0 and not file_too_large:
2435 print "Uploading %s file for %s" % (type, filename)
2436 url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
2437 form_fields = [("filename", filename),
2439 ("checksum", checksum),
2440 ("is_binary", str(is_binary)),
2441 ("is_current", str(not is_base)),
2444 form_fields.append(("file_too_large", "1"))
2446 form_fields.append(("user", options.email))
2447 ctype, body = EncodeMultipartFormData(form_fields,
2448 [("data", filename, content)])
2449 response_body = rpc_server.Send(url, body,
2451 if not response_body.startswith("OK"):
2452 StatusUpdate(" --> %s" % response_body)
2456 [patches.setdefault(v, k) for k, v in patch_list]
2457 for filename in patches.keys():
2458 base_content, new_content, is_binary, status = files[filename]
2459 file_id_str = patches.get(filename)
2460 if file_id_str.find("nobase") != -1:
2462 file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
2463 file_id = int(file_id_str)
2464 if base_content != None:
2465 UploadFile(filename, file_id, base_content, is_binary, status, True)
2466 if new_content != None:
2467 UploadFile(filename, file_id, new_content, is_binary, status, False)
2469 def IsImage(self, filename):
2470 """Returns true if the filename has an image extension."""
2471 mimetype = mimetypes.guess_type(filename)[0]
2474 return mimetype.startswith("image/")
2476 def IsBinary(self, filename):
2477 """Returns true if the guessed mimetyped isnt't in text group."""
2478 mimetype = mimetypes.guess_type(filename)[0]
2480 return False # e.g. README, "real" binaries usually have an extension
2481 # special case for text files which don't start with text/
2482 if mimetype in TEXT_MIMETYPES:
2484 return not mimetype.startswith("text/")
2487 class SubversionVCS(VersionControlSystem):
2488 """Implementation of the VersionControlSystem interface for Subversion."""
2490 def __init__(self, options):
2491 super(SubversionVCS, self).__init__(options)
2492 if self.options.revision:
2493 match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
2495 ErrorExit("Invalid Subversion revision %s." % self.options.revision)
2496 self.rev_start = match.group(1)
2497 self.rev_end = match.group(3)
2499 self.rev_start = self.rev_end = None
2500 # Cache output from "svn list -r REVNO dirname".
2501 # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
2502 self.svnls_cache = {}
2503 # SVN base URL is required to fetch files deleted in an older revision.
2504 # Result is cached to not guess it over and over again in GetBaseFile().
2505 required = self.options.download_base or self.options.revision is not None
2506 self.svn_base = self._GuessBase(required)
2508 def GuessBase(self, required):
2509 """Wrapper for _GuessBase."""
2510 return self.svn_base
2512 def _GuessBase(self, required):
2513 """Returns the SVN base URL.
2516 required: If true, exits if the url can't be guessed, otherwise None is
2519 info = RunShell(["svn", "info"])
2520 for line in info.splitlines():
2521 words = line.split()
2522 if len(words) == 2 and words[0] == "URL:":
2524 scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
2525 username, netloc = urllib.splituser(netloc)
2527 logging.info("Removed username from base URL")
2528 if netloc.endswith("svn.python.org"):
2529 if netloc == "svn.python.org":
2530 if path.startswith("/projects/"):
2532 elif netloc != "pythondev@svn.python.org":
2533 ErrorExit("Unrecognized Python URL: %s" % url)
2534 base = "http://svn.python.org/view/*checkout*%s/" % path
2535 logging.info("Guessed Python base = %s", base)
2536 elif netloc.endswith("svn.collab.net"):
2537 if path.startswith("/repos/"):
2539 base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
2540 logging.info("Guessed CollabNet base = %s", base)
2541 elif netloc.endswith(".googlecode.com"):
2543 base = urlparse.urlunparse(("http", netloc, path, params,
2545 logging.info("Guessed Google Code base = %s", base)
2548 base = urlparse.urlunparse((scheme, netloc, path, params,
2550 logging.info("Guessed base = %s", base)
2553 ErrorExit("Can't find URL in output from svn info")
2556 def GenerateDiff(self, args):
2557 cmd = ["svn", "diff"]
2558 if self.options.revision:
2559 cmd += ["-r", self.options.revision]
2561 data = RunShell(cmd)
2563 for line in data.splitlines():
2564 if line.startswith("Index:") or line.startswith("Property changes on:"):
2568 ErrorExit("No valid patches found in output from svn diff")
2571 def _CollapseKeywords(self, content, keyword_str):
2572 """Collapses SVN keywords."""
2573 # svn cat translates keywords but svn diff doesn't. As a result of this
2574 # behavior patching.PatchChunks() fails with a chunk mismatch error.
2575 # This part was originally written by the Review Board development team
2576 # who had the same problem (http://reviews.review-board.org/r/276/).
2577 # Mapping of keywords to known aliases
2580 'Date': ['Date', 'LastChangedDate'],
2581 'Revision': ['Revision', 'LastChangedRevision', 'Rev'],
2582 'Author': ['Author', 'LastChangedBy'],
2583 'HeadURL': ['HeadURL', 'URL'],
2587 'LastChangedDate': ['LastChangedDate', 'Date'],
2588 'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
2589 'LastChangedBy': ['LastChangedBy', 'Author'],
2590 'URL': ['URL', 'HeadURL'],
2595 return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
2596 return "$%s$" % m.group(1)
2598 for name in keyword_str.split(" ")
2599 for keyword in svn_keywords.get(name, [])]
2600 return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
2602 def GetUnknownFiles(self):
2603 status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
2605 for line in status.split("\n"):
2606 if line and line[0] == "?":
2607 unknown_files.append(line)
2608 return unknown_files
2610 def ReadFile(self, filename):
2611 """Returns the contents of a file."""
2612 file = open(filename, 'rb')
2615 result = file.read()
2620 def GetStatus(self, filename):
2621 """Returns the status of a file."""
2622 if not self.options.revision:
2623 status = RunShell(["svn", "status", "--ignore-externals", filename])
2625 ErrorExit("svn status returned no output for %s" % filename)
2626 status_lines = status.splitlines()
2627 # If file is in a cl, the output will begin with
2628 # "\n--- Changelist 'cl_name':\n". See
2629 # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
2630 if (len(status_lines) == 3 and
2631 not status_lines[0] and
2632 status_lines[1].startswith("--- Changelist")):
2633 status = status_lines[2]
2635 status = status_lines[0]
2636 # If we have a revision to diff against we need to run "svn list"
2637 # for the old and the new revision and compare the results to get
2638 # the correct status for a file.
2640 dirname, relfilename = os.path.split(filename)
2641 if dirname not in self.svnls_cache:
2642 cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
2643 out, returncode = RunShellWithReturnCode(cmd)
2645 ErrorExit("Failed to get status for %s." % filename)
2646 old_files = out.splitlines()
2647 args = ["svn", "list"]
2649 args += ["-r", self.rev_end]
2650 cmd = args + [dirname or "."]
2651 out, returncode = RunShellWithReturnCode(cmd)
2653 ErrorExit("Failed to run command %s" % cmd)
2654 self.svnls_cache[dirname] = (old_files, out.splitlines())
2655 old_files, new_files = self.svnls_cache[dirname]
2656 if relfilename in old_files and relfilename not in new_files:
2658 elif relfilename in old_files and relfilename in new_files:
2664 def GetBaseFile(self, filename):
2665 status = self.GetStatus(filename)
2669 # If a file is copied its status will be "A +", which signifies
2670 # "addition-with-history". See "svn st" for more information. We need to
2671 # upload the original file or else diff parsing will fail if the file was
2673 if status[0] == "A" and status[3] != "+":
2674 # We'll need to upload the new content if we're adding a binary file
2675 # since diff's output won't contain it.
2676 mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
2679 is_binary = bool(mimetype) and not mimetype.startswith("text/")
2680 if is_binary and self.IsImage(filename):
2681 new_content = self.ReadFile(filename)
2682 elif (status[0] in ("M", "D", "R") or
2683 (status[0] == "A" and status[3] == "+") or # Copied file.
2684 (status[0] == " " and status[1] == "M")): # Property change.
2686 if self.options.revision:
2687 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2689 # Don't change filename, it's needed later.
2691 args += ["-r", "BASE"]
2692 cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
2693 mimetype, returncode = RunShellWithReturnCode(cmd)
2695 # File does not exist in the requested revision.
2696 # Reset mimetype, it contains an error message.
2699 is_binary = bool(mimetype) and not mimetype.startswith("text/")
2700 if status[0] == " ":
2701 # Empty base content just to force an upload.
2704 if self.IsImage(filename):
2706 if status[0] == "M":
2707 if not self.rev_end:
2708 new_content = self.ReadFile(filename)
2710 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
2711 new_content = RunShell(["svn", "cat", url],
2712 universal_newlines=True, silent_ok=True)
2720 universal_newlines = False
2722 universal_newlines = True
2724 # "svn cat -r REV delete_file.txt" doesn't work. cat requires
2725 # the full URL with "@REV" appended instead of using "-r" option.
2726 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2727 base_content = RunShell(["svn", "cat", url],
2728 universal_newlines=universal_newlines,
2731 base_content = RunShell(["svn", "cat", filename],
2732 universal_newlines=universal_newlines,
2737 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
2740 args += ["-r", "BASE"]
2741 cmd = ["svn"] + args + ["propget", "svn:keywords", url]
2742 keywords, returncode = RunShellWithReturnCode(cmd)
2743 if keywords and not returncode:
2744 base_content = self._CollapseKeywords(base_content, keywords)
2746 StatusUpdate("svn status returned unexpected output: %s" % status)
2748 return base_content, new_content, is_binary, status[0:5]
2751 class GitVCS(VersionControlSystem):
2752 """Implementation of the VersionControlSystem interface for Git."""
2754 def __init__(self, options):
2755 super(GitVCS, self).__init__(options)
2756 # Map of filename -> (hash before, hash after) of base file.
2757 # Hashes for "no such file" are represented as None.
2759 # Map of new filename -> old filename for renames.
2762 def GenerateDiff(self, extra_args):
2763 # This is more complicated than svn's GenerateDiff because we must convert
2764 # the diff output to include an svn-style "Index:" line as well as record
2765 # the hashes of the files, so we can upload them along with our diff.
2767 # Special used by git to indicate "no such content".
2770 extra_args = extra_args[:]
2771 if self.options.revision:
2772 extra_args = [self.options.revision] + extra_args
2773 extra_args.append('-M')
2775 # --no-ext-diff is broken in some versions of Git, so try to work around
2776 # this by overriding the environment (but there is still a problem if the
2777 # git config key "diff.external" is used).
2778 env = os.environ.copy()
2779 if 'GIT_EXTERNAL_DIFF' in env: del env['GIT_EXTERNAL_DIFF']
2780 gitdiff = RunShell(["git", "diff", "--no-ext-diff", "--full-index"]
2781 + extra_args, env=env)
2785 for line in gitdiff.splitlines():
2786 match = re.match(r"diff --git a/(.*) b/(.*)$", line)
2789 # Intentionally use the "after" filename so we can show renames.
2790 filename = match.group(2)
2791 svndiff.append("Index: %s\n" % filename)
2792 if match.group(1) != match.group(2):
2793 self.renames[match.group(2)] = match.group(1)
2795 # The "index" line in a git diff looks like this (long hashes elided):
2796 # index 82c0d44..b2cee3f 100755
2797 # We want to save the left hash, as that identifies the base file.
2798 match = re.match(r"index (\w+)\.\.(\w+)", line)
2800 before, after = (match.group(1), match.group(2))
2801 if before == NULL_HASH:
2803 if after == NULL_HASH:
2805 self.hashes[filename] = (before, after)
2806 svndiff.append(line + "\n")
2808 ErrorExit("No valid patches found in output from git diff")
2809 return "".join(svndiff)
2811 def GetUnknownFiles(self):
2812 status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
2814 return status.splitlines()
2816 def GetFileContent(self, file_hash, is_binary):
2817 """Returns the content of a file identified by its git hash."""
2818 data, retcode = RunShellWithReturnCode(["git", "show", file_hash],
2819 universal_newlines=not is_binary)
2821 ErrorExit("Got error status from 'git show %s'" % file_hash)
2824 def GetBaseFile(self, filename):
2825 hash_before, hash_after = self.hashes.get(filename, (None,None))
2828 is_binary = self.IsBinary(filename)
2831 if filename in self.renames:
2832 status = "A +" # Match svn attribute name for renames.
2833 if filename not in self.hashes:
2834 # If a rename doesn't change the content, we never get a hash.
2835 base_content = RunShell(["git", "show", filename])
2836 elif not hash_before:
2839 elif not hash_after:
2844 is_image = self.IsImage(filename)
2846 # Grab the before/after content if we need it.
2847 # We should include file contents if it's text or it's an image.
2848 if not is_binary or is_image:
2849 # Grab the base content if we don't have it already.
2850 if base_content is None and hash_before:
2851 base_content = self.GetFileContent(hash_before, is_binary)
2852 # Only include the "after" file if it's an image; otherwise it
2853 # it is reconstructed from the diff.
2854 if is_image and hash_after:
2855 new_content = self.GetFileContent(hash_after, is_binary)
2857 return (base_content, new_content, is_binary, status)
2860 class MercurialVCS(VersionControlSystem):
2861 """Implementation of the VersionControlSystem interface for Mercurial."""
2863 def __init__(self, options, repo_dir):
2864 super(MercurialVCS, self).__init__(options)
2865 # Absolute path to repository (we can be in a subdir)
2866 self.repo_dir = os.path.normpath(repo_dir)
2867 # Compute the subdir
2868 cwd = os.path.normpath(os.getcwd())
2869 assert cwd.startswith(self.repo_dir)
2870 self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
2871 if self.options.revision:
2872 self.base_rev = self.options.revision
2874 mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}'])
2876 self.base_rev = mqparent
2878 self.base_rev = RunShell(["hg", "parents", "-q"]).split(':')[1].strip()
2879 def _GetRelPath(self, filename):
2880 """Get relative path of a file according to the current directory,
2881 given its logical path in the repo."""
2882 assert filename.startswith(self.subdir), (filename, self.subdir)
2883 return filename[len(self.subdir):].lstrip(r"\/")
2885 def GenerateDiff(self, extra_args):
2886 # If no file specified, restrict to the current subdir
2887 extra_args = extra_args or ["."]
2888 cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
2889 data = RunShell(cmd, silent_ok=True)
2892 for line in data.splitlines():
2893 m = re.match("diff --git a/(\S+) b/(\S+)", line)
2895 # Modify line to make it look like as it comes from svn diff.
2896 # With this modification no changes on the server side are required
2897 # to make upload.py work with Mercurial repos.
2898 # NOTE: for proper handling of moved/copied files, we have to use
2899 # the second filename.
2900 filename = m.group(2)
2901 svndiff.append("Index: %s" % filename)
2902 svndiff.append("=" * 67)
2906 svndiff.append(line)
2908 ErrorExit("No valid patches found in output from hg diff")
2909 return "\n".join(svndiff) + "\n"
2911 def GetUnknownFiles(self):
2912 """Return a list of files unknown to the VCS."""
2914 status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
2917 for line in status.splitlines():
2918 st, fn = line.split(" ", 1)
2920 unknown_files.append(fn)
2921 return unknown_files
2923 def GetBaseFile(self, filename):
2924 # "hg status" and "hg cat" both take a path relative to the current subdir
2925 # rather than to the repo root, but "hg diff" has given us the full path
2930 oldrelpath = relpath = self._GetRelPath(filename)
2931 # "hg status -C" returns two lines for moved/copied files, one otherwise
2932 out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
2933 out = out.splitlines()
2934 # HACK: strip error message about missing file/directory if it isn't in
2936 if out[0].startswith('%s: ' % relpath):
2938 status, what = out[0].split(' ', 1)
2939 if len(out) > 1 and status == "A" and what == relpath:
2940 oldrelpath = out[1].strip()
2942 if ":" in self.base_rev:
2943 base_rev = self.base_rev.split(":", 1)[0]
2945 base_rev = self.base_rev
2947 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
2949 is_binary = "\0" in base_content # Mercurial's heuristic
2951 new_content = open(relpath, "rb").read()
2952 is_binary = is_binary or "\0" in new_content
2953 if is_binary and base_content:
2954 # Fetch again without converting newlines
2955 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
2956 silent_ok=True, universal_newlines=False)
2957 if not is_binary or not self.IsImage(relpath):
2959 return base_content, new_content, is_binary, status
2962 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
2963 def SplitPatch(data):
2964 """Splits a patch into separate pieces for each file.
2967 data: A string containing the output of svn diff.
2970 A list of 2-tuple (filename, text) where text is the svn diff output
2971 pertaining to filename.
2976 for line in data.splitlines(True):
2978 if line.startswith('Index:'):
2979 unused, new_filename = line.split(':', 1)
2980 new_filename = new_filename.strip()
2981 elif line.startswith('Property changes on:'):
2982 unused, temp_filename = line.split(':', 1)
2983 # When a file is modified, paths use '/' between directories, however
2984 # when a property is modified '\' is used on Windows. Make them the same
2985 # otherwise the file shows up twice.
2986 temp_filename = temp_filename.strip().replace('\\', '/')
2987 if temp_filename != filename:
2988 # File has property changes but no modifications, create a new diff.
2989 new_filename = temp_filename
2991 if filename and diff:
2992 patches.append((filename, ''.join(diff)))
2993 filename = new_filename
2996 if diff is not None:
2998 if filename and diff:
2999 patches.append((filename, ''.join(diff)))
3003 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
3004 """Uploads a separate patch for each file in the diff output.
3006 Returns a list of [patch_key, filename] for each file.
3008 patches = SplitPatch(data)
3010 for patch in patches:
3011 if len(patch[1]) > MAX_UPLOAD_SIZE:
3012 print ("Not uploading the patch for " + patch[0] +
3013 " because the file is too large.")
3015 form_fields = [("filename", patch[0])]
3016 if not options.download_base:
3017 form_fields.append(("content_upload", "1"))
3018 files = [("data", "data.diff", patch[1])]
3019 ctype, body = EncodeMultipartFormData(form_fields, files)
3020 url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
3021 print "Uploading patch for " + patch[0]
3022 response_body = rpc_server.Send(url, body, content_type=ctype)
3023 lines = response_body.splitlines()
3024 if not lines or lines[0] != "OK":
3025 StatusUpdate(" --> %s" % response_body)
3027 rv.append([lines[1], patch[0]])
3032 """Helper to guess the version control system.
3034 This examines the current directory, guesses which VersionControlSystem
3035 we're using, and returns an string indicating which VCS is detected.
3038 A pair (vcs, output). vcs is a string indicating which VCS was detected
3039 and is one of VCS_GIT, VCS_MERCURIAL, VCS_SUBVERSION, or VCS_UNKNOWN.
3040 output is a string containing any interesting output from the vcs
3041 detection routine, or None if there is nothing interesting.
3043 # Mercurial has a command to get the base directory of a repository
3044 # Try running it, but don't die if we don't have hg installed.
3045 # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
3047 out, returncode = RunShellWithReturnCode(["hg", "root"])
3049 return (VCS_MERCURIAL, out.strip())
3050 except OSError, (errno, message):
3051 if errno != 2: # ENOENT -- they don't have hg installed.
3054 # Subversion has a .svn in all working directories.
3055 if os.path.isdir('.svn'):
3056 logging.info("Guessed VCS = Subversion")
3057 return (VCS_SUBVERSION, None)
3059 # Git has a command to test if you're in a git tree.
3060 # Try running it, but don't die if we don't have git installed.
3062 out, returncode = RunShellWithReturnCode(["git", "rev-parse",
3063 "--is-inside-work-tree"])
3065 return (VCS_GIT, None)
3066 except OSError, (errno, message):
3067 if errno != 2: # ENOENT -- they don't have git installed.
3070 return (VCS_UNKNOWN, None)
3073 def GuessVCS(options):
3074 """Helper to guess the version control system.
3076 This verifies any user-specified VersionControlSystem (by command line
3077 or environment variable). If the user didn't specify one, this examines
3078 the current directory, guesses which VersionControlSystem we're using,
3079 and returns an instance of the appropriate class. Exit with an error
3080 if we can't figure it out.
3083 A VersionControlSystem instance. Exits if the VCS can't be guessed.
3087 vcs = os.environ.get("CODEREVIEW_VCS")
3089 v = VCS_ABBREVIATIONS.get(vcs.lower())
3091 ErrorExit("Unknown version control system %r specified." % vcs)
3092 (vcs, extra_output) = (v, None)
3094 (vcs, extra_output) = GuessVCSName()
3096 if vcs == VCS_MERCURIAL:
3097 if extra_output is None:
3098 extra_output = RunShell(["hg", "root"]).strip()
3099 return MercurialVCS(options, extra_output)
3100 elif vcs == VCS_SUBVERSION:
3101 return SubversionVCS(options)
3102 elif vcs == VCS_GIT:
3103 return GitVCS(options)
3105 ErrorExit(("Could not guess version control system. "
3106 "Are you in a working copy directory?"))
3109 def RealMain(argv, data=None):
3110 """The real main function.
3113 argv: Command line arguments.
3114 data: Diff contents. If None (default) the diff is generated by
3115 the VersionControlSystem implementation returned by GuessVCS().
3118 A 2-tuple (issue id, patchset id).
3119 The patchset id is None if the base files are not uploaded by this
3120 script (applies only to SVN checkouts).
3122 logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
3123 "%(lineno)s %(message)s "))
3124 os.environ['LC_ALL'] = 'C'
3125 options, args = parser.parse_args(argv[1:])
3127 verbosity = options.verbose
3129 logging.getLogger().setLevel(logging.DEBUG)
3130 elif verbosity >= 2:
3131 logging.getLogger().setLevel(logging.INFO)
3132 vcs = GuessVCS(options)
3133 if isinstance(vcs, SubversionVCS):
3134 # base field is only allowed for Subversion.
3135 # Note: Fetching base files may become deprecated in future releases.
3136 base = vcs.GuessBase(options.download_base)
3139 if not base and options.download_base:
3140 options.download_base = True
3141 logging.info("Enabled upload of base file")
3142 if not options.assume_yes:
3143 vcs.CheckForUnknownFiles()
3145 data = vcs.GenerateDiff(args)
3146 files = vcs.GetBaseFiles(data)
3148 print "Upload server:", options.server, "(change with -s/--server)"
3150 prompt = "Message describing this patch set: "
3152 prompt = "New issue subject: "
3153 message = options.message or raw_input(prompt).strip()
3155 ErrorExit("A non-empty message is required")
3156 rpc_server = GetRpcServer(options)
3157 form_fields = [("subject", message)]
3159 form_fields.append(("base", base))
3161 form_fields.append(("issue", str(options.issue)))
3163 form_fields.append(("user", options.email))
3164 if options.reviewers:
3165 for reviewer in options.reviewers.split(','):
3166 if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
3167 ErrorExit("Invalid email address: %s" % reviewer)
3168 form_fields.append(("reviewers", options.reviewers))
3170 for cc in options.cc.split(','):
3171 if "@" in cc and not cc.split("@")[1].count(".") == 1:
3172 ErrorExit("Invalid email address: %s" % cc)
3173 form_fields.append(("cc", options.cc))
3174 description = options.description
3175 if options.description_file:
3176 if options.description:
3177 ErrorExit("Can't specify description and description_file")
3178 file = open(options.description_file, 'r')
3179 description = file.read()
3182 form_fields.append(("description", description))
3183 # Send a hash of all the base file so the server can determine if a copy
3184 # already exists in an earlier patchset.
3186 for file, info in files.iteritems():
3187 if not info[0] is None:
3188 checksum = md5(info[0]).hexdigest()
3191 base_hashes += checksum + ":" + file
3192 form_fields.append(("base_hashes", base_hashes))
3195 print "Warning: Private flag ignored when updating an existing issue."
3197 form_fields.append(("private", "1"))
3198 # If we're uploading base files, don't send the email before the uploads, so
3199 # that it contains the file status.
3200 if options.send_mail and options.download_base:
3201 form_fields.append(("send_mail", "1"))
3202 if not options.download_base:
3203 form_fields.append(("content_upload", "1"))
3204 if len(data) > MAX_UPLOAD_SIZE:
3205 print "Patch is large, so uploading file patches separately."
3206 uploaded_diff_file = []
3207 form_fields.append(("separate_patches", "1"))
3209 uploaded_diff_file = [("data", "data.diff", data)]
3210 ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
3211 response_body = rpc_server.Send("/upload", body, content_type=ctype)
3213 if not options.download_base or not uploaded_diff_file:
3214 lines = response_body.splitlines()
3217 patchset = lines[1].strip()
3218 patches = [x.split(" ", 1) for x in lines[2:]]
3223 if not response_body.startswith("Issue created.") and \
3224 not response_body.startswith("Issue updated."):
3225 print >>sys.stderr, msg
3227 issue = msg[msg.rfind("/")+1:]
3229 if not uploaded_diff_file:
3230 result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
3231 if not options.download_base:
3234 if not options.download_base:
3235 vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
3236 if options.send_mail:
3237 rpc_server.Send("/" + issue + "/mail", payload="")
3238 return issue, patchset
3244 except KeyboardInterrupt:
3246 StatusUpdate("Interrupted.")